nova卸载volume源码分析

基于llvm +iscsi协议进行分析

1、客戶端接受请求并路由到 VolumeAttachmentController

其对应的restfull请求格式如下：
delete /servers/{server_id}/os-volume_attachments/{volume_id}

nova-api处理该请求的入口函数为 nova.api.openstack.compute.volumes.VolumeAttachmentController.delete

nova/api/openstack/compute/volumes.py
from nova.compute import api as compute
class VolumeAttachmentController(wsgi.Controller):
    def __init__(self):
        self.compute_api = compute.API()
        self.volume_api = cinder.API()
        super(VolumeAttachmentController, self).__init__()
		
    @wsgi.response(202)
    @wsgi.expected_errors((400, 403, 404, 409))
    def delete(self, req, server_id, id):
        """Detach a volume from an instance."""
        context = req.environ['nova.context']
        context.can(va_policies.POLICY_ROOT % 'delete')
        volume_id = id
        instance = common.get_instance(self.compute_api, context, server_id,-------根据虚机uuid获取instance实例对象
                                       expected_attrs=['device_metadata'])
        if instance.vm_state in (vm_states.SHELVED,
                                 vm_states.SHELVED_OFFLOADED):
            _check_request_version(req, '2.20', 'detach_volume',
                                   server_id, instance.vm_state)
        try:
            volume = self.volume_api.get(context, volume_id)-----------调用cinderclient，根据卷uuid获取卷实例对象
        except exception.VolumeNotFound as e:
            raise exc.HTTPNotFound(explanation=e.format_message())

        try:
            bdm = objects.BlockDeviceMapping.get_by_volume_and_instance(-------获取block_device_mapping表 中该 instance挂在该卷的映射信息
                context, volume_id, instance.uuid)
        except exception.VolumeBDMNotFound:
            msg = (_("Instance %(instance)s is not attached "
                     "to volume %(volume)s") %
                   {'instance': server_id, 'volume': volume_id})
            raise exc.HTTPNotFound(explanation=msg)

        if bdm.is_root:
            msg = _("Cannot detach a root device volume")
            raise exc.HTTPBadRequest(explanation=msg)

        try:
            self.compute_api.detach_volume(context, instance, volume)----s1 nova-api处理过程
			.....

s1 nova-api处理过程详解

nova api服务调用nova-compute服务对外的呈现接口

nova.compute.api.API.detach_volume
nova/compute/api.py
class API(base.Base):
    def detach_volume(self, context, instance, volume):
        """Detach a volume from an instance."""
        if instance.vm_state == vm_states.SHELVED_OFFLOADED:
            self._detach_volume_shelved_offloaded(context, instance, volume)
        else:
            self._detach_volume(context, instance, volume)-----一般情况走这个分支

    def _detach_volume(self, context, instance, volume):
        """Detach volume from instance.
        This method is separated to make it easier for cells version
        to override.
        """
        try:
            self.volume_api.begin_detaching(context, volume['id'])------调用cinderclient，更新cinder数据库中卷的状态为detaching
        except exception.InvalidInput as exc:
            raise exception.InvalidVolume(reason=exc.format_message())
        attachments = volume.get('attachments', {})
        attachment_id = None
        if attachments and instance.uuid in attachments:
            attachment_id = attachments[instance.uuid]['attachment_id']
        self._record_action_start(context, instance, instance_actions.DETACH_VOLUME)-----记录对虚机的一次操作
        self.compute_rpcapi.detach_volume(context, instance=instance,------调用nova-compute rpc客户端发送卸载卷的rpc请求
                volume_id=volume['id'], attachment_id=attachment_id)	

2、nova-compute服务接受发送过来的卸载卷rpc请求并处理，其处理入口函数为

nova/compute/manager.py
class ComputeManager(manager.Manager):

    def detach_volume(self, context, volume_id, instance, attachment_id):
        """Detach a volume from an instance.
        :param context: security context
        :param volume_id: the volume id
        :param instance: the Instance object to detach the volume from
        :param attachment_id: The volume attachment_id for the given instance
                              and volume.
        """
        @utils.synchronized(instance.uuid)
        def do_detach_volume(context, volume_id, instance, attachment_id):
            bdm = objects.BlockDeviceMapping.get_by_volume_and_instance(
                    context, volume_id, instance.uuid)
            self._detach_volume(context, bdm, instance,
                                attachment_id=attachment_id)

        do_detach_volume(context, volume_id, instance, attachment_id)----实际调用的是 _detach_volume

   def _detach_volume(self, context, bdm, instance, destroy_bdm=True,
                       attachment_id=None):
        """Detach a volume from an instance.
        """
        volume_id = bdm.volume_id
        compute_utils.notify_about_volume_attach_detach(
            context, instance, self.host,
            action=fields.NotificationAction.VOLUME_DETACH,
            phase=fields.NotificationPhase.START,
            volume_id=volume_id)

        self._notify_volume_usage_detach(context, instance, bdm)

        LOG.info('Detaching volume %(volume_id)s',
                 {'volume_id': volume_id}, instance=instance)

        driver_bdm = driver_block_device.convert_volume(bdm)------获取bdm driver驱动，该参数的值为nova.virt.block_device.DriverVolumeBlockDevice
        driver_bdm.detach(context, instance, self.volume_api, self.driver,-------- s1 调用bdm 驱动执行卸载卷操作，self.driver = nova.virt.libvirt.driver.LibvirtDriver
                          attachment_id=attachment_id, destroy_bdm=destroy_bdm)
        info = dict(volume_id=volume_id)
        self._notify_about_instance_usage(
            context, instance, "volume.detach", extra_usage_info=info)
        compute_utils.notify_about_volume_attach_detach(
            context, instance, self.host,
            action=fields.NotificationAction.VOLUME_DETACH,
            phase=fields.NotificationPhase.END,
            volume_id=volume_id)

        if 'tag' in bdm and bdm.tag:
            self._delete_disk_metadata(instance, bdm)
        if destroy_bdm:
            bdm.destroy()-----设置nova 数据库中该卷的bdm deleted标志位删除状态

s1 BDM driver detach操作

nova/virt/block_device.py
class DriverVolumeBlockDevice(DriverBlockDevice):
   def detach(self, context, instance, volume_api, virt_driver,
               attachment_id=None, destroy_bdm=False):
		  
        volume = self._get_volume(context, volume_api, self.volume_id)
        if volume.get('shared_targets', False):
            # Lock the detach call using the provided service_uuid.
            @utils.synchronized(volume['service_uuid'])
            def _do_locked_detach(*args, **_kwargs):
                self._do_detach(*args, **_kwargs)
            _do_locked_detach(context, instance, volume_api, virt_driver,
                              attachment_id, destroy_bdm)
        else:
            # We don't need to (or don't know if we need to) lock.
            self._do_detach(context, instance, volume_api, virt_driver,-----调试走了该分支
                            attachment_id, destroy_bdm)


    def _do_detach(self, context, instance, volume_api, virt_driver,
                   attachment_id=None, destroy_bdm=False):
        """Private method that actually does the detach.

        This is separate from the detach() method so the caller can optionally
        lock this call.
        """
        volume_id = self.volume_id

        # Only attempt to detach and disconnect from the volume if the instance
        # is currently associated with the local compute host.
        if CONF.host == instance.host:
            self.driver_detach(context, instance, volume_api, virt_driver)--------s1.1 虚机端卸载卷操作
        elif not destroy_bdm:
            LOG.debug("Skipping driver_detach during remote rebuild.",
                      instance=instance)
        elif destroy_bdm:
            LOG.error("Unable to call for a driver detach of volume "
                      "%(vol_id)s due to the instance being "
                      "registered to the remote host %(inst_host)s.",
                      {'vol_id': volume_id,
                       'inst_host': instance.host}, instance=instance)

        # NOTE(jdg): For now we need to actually inspect the bdm for an
        # attachment_id as opposed to relying on what may have been passed
        # in, we want to force usage of the old detach flow for now and only
        # use the new flow when we explicitly used it for the attach.
        if not self['attachment_id']:
            connector = virt_driver.get_volume_connector(instance)
            connection_info = self['connection_info']
            if connection_info and not destroy_bdm and (
               connector.get('host') != instance.host):
                # If the volume is attached to another host (evacuate) then
                # this connector is for the wrong host. Use the connector that
                # was stored in connection_info instead (if we have one, and it
                # is for the expected host).
                stashed_connector = connection_info.get('connector')
                if not stashed_connector:
                    # Volume was attached before we began stashing connectors
                    LOG.warning("Host mismatch detected, but stashed "
                                "volume connector not found. Instance host is "
                                "%(ihost)s, but volume connector host is "
                                "%(chost)s.",
                                {'ihost': instance.host,
                                 'chost': connector.get('host')})
                elif stashed_connector.get('host') != instance.host:
                    # Unexpected error. The stashed connector is also not
                    # matching the needed instance host.
                    LOG.error("Host mismatch detected in stashed volume "
                              "connector. Will use local volume connector. "
                              "Instance host is %(ihost)s. Local volume "
                              "connector host is %(chost)s. Stashed volume "
                              "connector host is %(schost)s.",
                              {'ihost': instance.host,
                               'chost': connector.get('host'),
                               'schost': stashed_connector.get('host')})
                else:
                    # Fix found. Use stashed connector.
                    LOG.debug("Host mismatch detected. Found usable stashed "
                              "volume connector. Instance host is %(ihost)s. "
                              "Local volume connector host was %(chost)s. "
                              "Stashed volume connector host is %(schost)s.",
                              {'ihost': instance.host,
                               'chost': connector.get('host'),
                               'schost': stashed_connector.get('host')})
                    connector = stashed_connector

            volume_api.terminate_connection(context, volume_id, connector)-----s1.2 调用cinderclient，发送os-terminate_connection请求，cinder端取消后端存储卷的挂载关系
            volume_api.detach(context.elevated(), volume_id, instance.uuid,attachment_id)-----s1.3 调用cinderclient，发送os-detach请求，更新cinder 数据库中，卷的状态
        else:
            volume_api.attachment_delete(context, self['attachment_id'])

s1.1 虚机端卸载卷操作
self.driver_detach(context, instance, volume_api, virt_driver)	
nova/virt/block_device.py	
class DriverVolumeBlockDevice(DriverBlockDevice):
    def driver_detach(self, context, instance, volume_api, virt_driver):
	#virt_driver的值为	nova.virt.libvirt.driver.LibvirtDriver
        connection_info = self['connection_info']
        mp = self['mount_device']
        volume_id = self.volume_id

        LOG.info('Attempting to driver detach volume %(volume_id)s from '
                 'mountpoint %(mp)s', {'volume_id': volume_id, 'mp': mp},
                 instance=instance)
        try:
            if not virt_driver.instance_exists(instance):
                LOG.warning('Detaching volume from unknown instance',
                            instance=instance)

            encryption = encryptors.get_encryption_metadata(context,
                    volume_api, volume_id, connection_info)
            virt_driver.detach_volume(context, connection_info, instance, mp,------------s1.1.1 调用libvirt里面的实际驱动进行处理
                                      encryption=encryption)
        except exception.DiskNotFound as err:
            LOG.warning('Ignoring DiskNotFound exception while '
                        'detaching volume %(volume_id)s from '
                        '%(mp)s : %(err)s',
                        {'volume_id': volume_id, 'mp': mp,
                         'err': err}, instance=instance)
        except exception.DeviceDetachFailed as err:
            with excutils.save_and_reraise_exception():
                LOG.warning('Guest refused to detach volume %(vol)s',
                            {'vol': volume_id}, instance=instance)
                volume_api.roll_detaching(context, volume_id)
        except Exception:
            with excutils.save_and_reraise_exception():
                LOG.exception('Failed to detach volume '
                              '%(volume_id)s from %(mp)s',
                              {'volume_id': volume_id, 'mp': mp},
                              instance=instance)
                volume_api.roll_detaching(context, volume_id)
s1.1.1 详解
nova/virt/libvirt/driver.py
class LibvirtDriver(driver.ComputeDriver):
    def detach_volume(self, context, connection_info, instance, mountpoint,
                      encryption=None):
        disk_dev = mountpoint.rpartition("/")[2]
        try:
            guest = self._host.get_guest(instance)-------获取虚机的xml信息
            state = guest.get_power_state(self._host)
            live = state in (power_state.RUNNING, power_state.PAUSED)
            wait_for_detach = guest.detach_device_with_retry(guest.get_disk,disk_dev,live=live)----s1删除xml文件中挂载卷的信息
            wait_for_detach()
        self._disconnect_volume(context, connection_info, instance,-------s2主机端执行issci logout操作
                                encryption=encryption)
s2主机端执行issci logout操作
nova/virt/libvirt/driver.py
class LibvirtDriver(driver.ComputeDriver):
    def _disconnect_volume(self, context, connection_info, instance,
                           encryption=None):
        self._detach_encryptor(context, connection_info, encryption=encryption)
        if self._should_disconnect_target(context, connection_info, instance):
            vol_driver = self._get_volume_driver(connection_info)
            vol_driver.disconnect_volume(connection_info, instance)
        else:
            LOG.info("Detected multiple connections on this host for volume: "
                     "%s, skipping target disconnect.",
                     driver_block_device.get_volume_id(connection_info),
                     instance=instance)
根据block_device_mapping中，connection_info的 driver_volume_type 类型来获取对应的驱动，
由于使用的是iscsi协议，因此找的 LibvirtISCSIVolumeDriver
最终走的是vol_driver.disconnect_volume=nova.virt.libvirt.volume.iscsi.LibvirtISCSIVolumeDriver.disconnect_volume