k8s volume plugin 源码分析: Local卷 MountDevice 实现

func (og *operationGenerator) GenerateMountVolumeFunc(...) volumetypes.GeneratedOperations {
    ...
    mountVolumeFunc := func() volumetypes.OperationContext {
        ...
        deviceMountableVolumePlugin, _ := og.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeToMount.VolumeSpec)
        var volumeDeviceMounter volume.DeviceMounter
        if deviceMountableVolumePlugin != nil {
            volumeDeviceMounter, _ = deviceMountableVolumePlugin.NewDeviceMounter()
        }
        ...
        if volumeDeviceMounter != nil && actualStateOfWorld.GetDeviceMountState(volumeToMount.VolumeName) != DeviceGloballyMounted {
            deviceMountPath, err :=
                volumeDeviceMounter.GetDeviceMountPath(volumeToMount.VolumeSpec)
            ...
            // Mount device to global mount path
            err = volumeDeviceMounter.MountDevice(
                volumeToMount.VolumeSpec,
                devicePath,
                deviceMountPath,
                volume.DeviceMounterArgs{FsGroup: fsGroup},
            )
            ...
            // Update actual state of world to reflect volume is globally mounted
            markDeviceMountedErr := actualStateOfWorld.MarkDeviceAsMounted(
                volumeToMount.VolumeName, devicePath, deviceMountPath)
            ...

            // If volume expansion is performed after MountDevice but before SetUp then
            // deviceMountPath and deviceStagePath is going to be the same.
            // Deprecation: Calling NodeExpandVolume after NodeStage/MountDevice will be deprecated
            // in a future version of k8s.
            resizeOptions.DeviceMountPath = deviceMountPath
            resizeOptions.DeviceStagePath = deviceMountPath
            resizeOptions.CSIVolumePhase = volume.CSIVolumeStaged

            // NodeExpandVolume will resize the file system if user has requested a resize of
            // underlying persistent volume and is allowed to do so.
            resizeDone, resizeError = og.nodeExpandVolume(volumeToMount, actualStateOfWorld, resizeOptions)
            ...
        }        
        ...
    }
}

func (dm *deviceMounter) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, _ volume.DeviceMounterArgs) error {
	...
	fileType, err := dm.hostUtil.GetFileType(spec.PersistentVolume.Spec.Local.Path)
	if err != nil {
		return err
	}

	switch fileType {
	case hostutil.FileTypeBlockDev:
		// local volume plugin does not implement AttachableVolumePlugin interface, so set devicePath to Path in PV spec directly
		return dm.mountLocalBlockDevice(spec, spec.PersistentVolume.Spec.Local.Path, deviceMountPath)
	case hostutil.FileTypeDirectory:
		// if the given local volume path is of already filesystem directory, return directly
		return nil
	default:
		return fmt.Errorf("only directory and block device are supported")
	}
}

func (dm *deviceMounter) mountLocalBlockDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error {
    ...
    notMnt, err := dm.mounter.IsLikelyNotMountPoint(deviceMountPath)
    if err != nil {
        if os.IsNotExist(err) {
            if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
                return err
            }
            ...
        } 
        ...
    }
    ...
    err = dm.mounter.FormatAndMount(devicePath, deviceMountPath, fstype, mountOptions)
    if err != nil {
        ...
        return ...
    }
    ...
    return nil
}

func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error {
    return mounter.FormatAndMountSensitive(source, target, fstype, options, nil /* sensitiveOptions */)
}

func (mounter *SafeFormatAndMount) FormatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
    return mounter.formatAndMountSensitive(source, target, fstype, options, sensitiveOptions)
}

func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
    ...
}

func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
    ...
    // Check if the disk is already formatted
    existingFormat, err := mounter.GetDiskFormat(source)
    if err != nil {
        return NewMountError(GetDiskFormatFailed, "failed to get disk format of disk %s: %v", source, err)
    }
    ...
}

func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) {
    return getDiskFormat(mounter.Exec, disk)
}

func getDiskFormat(exec utilexec.Interface, disk string) (string, error) {
    args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk}
    klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args)
    dataOut, err := exec.Command("blkid", args...).CombinedOutput()
    output := string(dataOut)
    ...
    var fstype, pttype string

    lines := strings.Split(output, "\n")
    for _, l := range lines {
        ...
        cs := strings.Split(l, "=")
        ...
        // TYPE is filesystem type, and PTTYPE is partition table type, according
        // to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/.
        if cs[0] == "TYPE" {
            fstype = cs[1]
        } else if cs[0] == "PTTYPE" {
            pttype = cs[1]
        }
    }
    ...
    return fstype, nil
}

func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
    ...
    existingFormat, err := mounter.GetDiskFormat(source)
    ...
    // Use 'ext4' as the default
    if len(fstype) == 0 {
        fstype = "ext4"
    }

    if existingFormat == "" {
        // Do not attempt to format the disk if mounting as readonly, return an error to reflect this.
        if readOnly {
            return NewMountError(UnformattedReadOnly, "cannot mount unformatted disk %s as we are manipulating it in read-only mode", source)
        }

        // Disk is unformatted so format it.
        args := []string{source}
        if fstype == "ext4" || fstype == "ext3" {
            args = []string{
                "-F",  // Force flag
                "-m0", // Zero blocks reserved for super-user
                source,
            }
        } else if fstype == "xfs" {
            args = []string{
                "-f", // force flag
                source,
            }
        }

        klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args)
        output, err := mounter.Exec.Command("mkfs."+fstype, args...).CombinedOutput()
        if err != nil {
            ...
            return ...
        }
        ...
    } else {
        ...
    }
    ...
}

func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
    ...
    existingFormat, err := mounter.GetDiskFormat(source)
    ...
    // Use 'ext4' as the default
    if len(fstype) == 0 {
        fstype = "ext4"
    }
    if existingFormat == "" {
        ...
    } else {        
        if fstype != existingFormat {
            ...
        }
        if !readOnly {
            // Run check tools on the disk to fix repairable issues, only do this for formatted volumes requested as rw.
            err := mounter.checkAndRepairFilesystem(source)
            if err != nil {
                return err
            }
        }
    }

func (mounter *SafeFormatAndMount) checkAndRepairFilesystem(source string) error {
    klog.V(4).Infof("Checking for issues with fsck on disk: %s", source)
    args := []string{"-a", source}
    out, err := mounter.Exec.Command("fsck", args...).CombinedOutput()
    if err != nil {
        ee, isExitError := err.(utilexec.ExitError)
        switch {
        case err == utilexec.ErrExecutableNotFound:
            klog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.")
        case isExitError && ee.ExitStatus() == fsckErrorsCorrected:
            klog.Infof("Device %s has errors which were corrected by fsck.", source)
        case isExitError && ee.ExitStatus() == fsckErrorsUncorrected:
            return NewMountError(HasFilesystemErrors, "'fsck' found errors on device %s but could not correct them: %s", source, string(out))
        case isExitError && ee.ExitStatus() > fsckErrorsUncorrected:
            klog.Infof("`fsck` error %s", string(out))
        }
    }
    return nil
}

func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
    ...
    // Mount the disk
    klog.V(4).Infof("Attempting to mount disk %s in %s format at %s", source, fstype, target)
    if err := mounter.MountSensitive(source, target, fstype, options, sensitiveOptions); err != nil {
        return NewMountError(mountErrorValue, err.Error())
    }

    return nil
}

func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
    // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty.
    // All Linux distros are expected to be shipped with a mount utility that a support bind mounts.
    mounterPath := ""
    bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := MakeBindOptsSensitive(options, sensitiveOptions)
    if bind {
        err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive, nil /* mountFlags */, true)
        if err != nil {
            return err
        }
        return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive, nil /* mountFlags */, true)
    }
    ...
    return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, nil /* mountFlags */, true)
}

func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string, systemdMountRequired bool) error {
    mountArgs, mountArgsLogStr := MakeMountArgsSensitiveWithMountFlags(source, target, fstype, options, sensitiveOptions, mountFlags)
    if len(mounterPath) > 0 {
        mountArgs = append([]string{mountCmd}, mountArgs...)
        mountArgsLogStr = mountCmd + " " + mountArgsLogStr
        mountCmd = mounterPath
    }
    ...
    command := exec.Command(mountCmd, mountArgs...)
    output, err := command.CombinedOutput()
    ...
    return err
}

func MakeMountArgsSensitiveWithMountFlags(source, target, fstype string, options []string, sensitiveOptions []string, mountFlags []string) (mountArgs []string, mountArgsLogStr string) {
	// Build mount command as follows:
	//   mount [$mountFlags] [-t $fstype] [-o $options] [$source] $target
    mountArgs = []string{}
    ...
    mountArgs = append(mountArgs, mountFlags...)
    ...
    if len(fstype) > 0 {
        mountArgs = append(mountArgs, "-t", fstype)
        ...
    }
    if len(options) > 0 || len(sensitiveOptions) > 0 {
        combinedOptions := []string{}
        combinedOptions = append(combinedOptions, options...)
        combinedOptions = append(combinedOptions, sensitiveOptions...)
        mountArgs = append(mountArgs, "-o", strings.Join(combinedOptions, ","))
        ...
    }
    if len(source) > 0 {
        mountArgs = append(mountArgs, source)
        ...
    }
    mountArgs = append(mountArgs, target)
    ...
    return mountArgs, mountArgsLogStr
}