openstack--9--深入理解虚拟机

登录计算节点查看进程

[root@linux-node2 ~]# ps aux | grep kvm
root        824  0.0  0.0      0     0 ?        S<   10:19   0:00 [kvm-irqfd-clean]
qemu       9762 18.2  3.2 538924 131596 ?       Sl   22:11   0:59 /usr/libexec/qemu-kvm -name instance-00000002 -S -machine pc-i440fx-rhel7.0.0,accel=kvm,usb=off -cpu
 SandyBridge,+vme,+ds,+ss,+ht,+vmx,+pcid,+osxsave,+hypervisor,+arat,+tsc_adjust -m 64 -realtime mlock=off -smp 1,sockets=1,cores=1,threads=1 -uuid
 4e810c6c-5c56-4160-a29c-b240a4550679 -smbios type=1,manufacturer=Fedora Project,product=OpenStack
 Nova,version=13.1.2-1.el7,serial=df6d7bf8-67af-42d3-b535-d710f13b0201,uuid=4e810c6c-5c56-4160-a29c-b240a4550679,family=Virtual Machine -no-user-config -nodefaults
 -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-2-instance-00000002/monitor.sock,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc
 base=utc,driftfix=slew -global kv-pit.lost_tick_policy=discard -no-hpet -no-shutdown -boot strict=on -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -drive
 file=/var/lib/nova/instances/4e810c6c-5c56-4160-a29c-b240a4550679/disk,format=qcow2,if=none,id=drive-virtio-disk0,cache=none -device
 virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=26,id=hostnet0,vhost=on,vhostfd=28 -device
 virtio-net-pci,netdev=hostnet0,id=net0,mac=fa:16:3e:ff:a2:6e,bus=pci.0,addr=0x3 -chardev
 file,id=charserial0,path=/var/lib/nova/instances/4e810c6c-5c56-4160-a29c-b240a4550679/console.log -device isa-serial,chardev=charserial0,id=serial0 -chardev
 pty,id=charserial1 -device isa-serial,chardev=charserial1,id=serial1 -device usb-tablet,id=input0,bus=usb.0,port=1 -vnc 0.0.0.0:0 -k en-us -vga cirrus -device
 virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -msg timestamp=on
root       9781  0.0  0.0      0     0 ?        S    22:11   0:00 [kvm-pit/9762]
root       9903  0.0  0.0 112648   964 pts/0    S+   22:16   0:00 grep --colour=auto kvm
[root@linux-node2 ~]# 

　

查看端口监听

[root@linux-node2 ~]# netstat -lntp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
tcp        0      0 0.0.0.0:5900            0.0.0.0:*               LISTEN      9762/qemu-kvm       
tcp        0      0 0.0.0.0:111             0.0.0.0:*               LISTEN      1/systemd           
tcp        0      0 192.168.122.1:53        0.0.0.0:*               LISTEN      3916/dnsmasq        
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      1155/sshd           
tcp        0      0 127.0.0.1:25            0.0.0.0:*               LISTEN      1250/master         
tcp6       0      0 :::111                  :::*                    LISTEN      1/systemd           
tcp6       0      0 :::22                   :::*                    LISTEN      1155/sshd           
tcp6       0      0 ::1:25                  :::*                    LISTEN      1250/master         

　　

查看桥接

[root@linux-node2 ~]# brctl show
bridge name	bridge id		STP enabled	interfaces
brqac1b0655-93		8000.000c29f17278	no		eth0
							tap04087c6c-47
virbr0		8000.525400747276	yes		virbr0-nic
[root@linux-node2 ~]# 

　　

查看网络设备

[root@linux-node2 ~]# ifconfig
brqac1b0655-93: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.56.12  netmask 255.255.255.0  broadcast 192.168.56.255
        inet6 fe80::a846:99ff:fedb:caf3  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:f1:72:78  txqueuelen 0  (Ethernet)
        RX packets 23781  bytes 5440813 (5.1 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 34638  bytes 8215484 (7.8 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet6 fe80::20c:29ff:fef1:7278  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:f1:72:78  txqueuelen 1000  (Ethernet)
        RX packets 169659  bytes 109732624 (104.6 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 96397  bytes 29661554 (28.2 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

　　

登录控制节点查看

[root@linux-node1 ~]# openstack server list
+--------------------------------------+------+--------+---------------------------+
| ID                                   | Name | Status | Networks                  |
+--------------------------------------+------+--------+---------------------------+
| 4e810c6c-5c56-4160-a29c-b240a4550679 | demo | ACTIVE | public-net=192.168.56.102 |
+--------------------------------------+------+--------+---------------------------+
[root@linux-node1 ~]# 

　　

计算节点查看虚拟机存放路径

虚拟机在下面路径下，这里的名字和上面看到的是一致的

[root@linux-node2 ~]# cd /var/lib/nova/instances/
[root@linux-node2 instances]# ls
4e810c6c-5c56-4160-a29c-b240a4550679  _base  compute_nodes  locks
[root@linux-node2 instances]# cd 4e810c6c-5c56-4160-a29c-b240a4550679/
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls
console.log  disk  disk.info  libvirt.xml
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls -alh
total 2.3M
drwxr-xr-x 2 nova nova   69 Feb 18 22:11 .
drwxr-xr-x 5 nova nova   93 Feb 18 22:11 ..
-rw-rw---- 1 qemu qemu  19K Feb 18 22:11 console.log
-rw-r--r-- 1 qemu qemu 2.3M Feb 18 22:12 disk
-rw-r--r-- 1 nova nova   79 Feb 18 22:11 disk.info
-rw-r--r-- 1 nova nova 2.6K Feb 18 22:11 libvirt.xml
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# 

　　

下面这个就是镜像

[root@linux-node2 instances]# pwd
/var/lib/nova/instances
[root@linux-node2 instances]# cd _base/
[root@linux-node2 _base]# ls
8c31d8bd210f46f11ae77d077c33cff34e274fcb
[root@linux-node2 _base]# file 8c31d8bd210f46f11ae77d077c33cff34e274fcb 
8c31d8bd210f46f11ae77d077c33cff34e274fcb: x86 boot sector; GRand Unified Bootloader, stage1 version 0x3, stage2 address 0x2000, stage2 segment 0x200;
 partition 1: ID=0x83, active, starthead 0, startsector 16065, 64260 sectors, code offset 0x48
[root@linux-node2 _base]# 

　　

控制节点查看镜像

[root@linux-node1 ~]# openstack image list
+--------------------------------------+--------+--------+
| ID                                   | Name   | Status |
+--------------------------------------+--------+--------+
| 9969eaa3-0296-48cc-a42e-a02251b778a6 | cirros | active |
+--------------------------------------+--------+--------+
[root@linux-node1 ~]# cd /var/lib/glance/images/
[root@linux-node1 images]# ls
9969eaa3-0296-48cc-a42e-a02251b778a6
[root@linux-node1 images]# 

　　

我们创建的虚拟机很小，只有2.3M，其实它有个后端文件。40MB
qemu v3格式的，只保存文件变化的部分，剩下的全是一个镜像，这样节省空间，启动也很快

[root@linux-node2 instances]# pwd
/var/lib/nova/instances
[root@linux-node2 instances]# ls
4e810c6c-5c56-4160-a29c-b240a4550679  _base  compute_nodes  locks
[root@linux-node2 instances]# cd 4e810c6c-5c56-4160-a29c-b240a4550679/
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls
console.log  disk  disk.info  libvirt.xml
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls -lh disk
-rw-r--r-- 1 qemu qemu 2.3M Feb 18 22:12 disk
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# file disk
disk: QEMU QCOW Image (v3), has backing file (path /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34), 1073741824 bytes
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls -lh /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34
ls: cannot access /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34: No such file or directory
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls -lh /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34e274fcb 
-rw-r--r-- 1 qemu qemu 40M Feb 18 22:11 /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34e274fcb
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# 

　　

也可以通过下面方式查看磁盘信息

[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# cd -
/var/lib/nova/instances
[root@linux-node2 instances]# ls
4e810c6c-5c56-4160-a29c-b240a4550679  _base  compute_nodes  locks
[root@linux-node2 instances]# cd 4e810c6c-5c56-4160-a29c-b240a4550679/
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls
console.log  disk  disk.info  libvirt.xml
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# qemu-img info disk
image: disk
file format: qcow2
virtual size: 1.0G (1073741824 bytes)
disk size: 2.3M
cluster_size: 65536
backing file: /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34e274fcb
Format specific information:
    compat: 1.1
    lazy refcounts: false
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# 

　　

虚拟机的xml文件在下面两个位置都能看到，它是动态生成的

[root@linux-node2 qemu]# pwd
/etc/libvirt/qemu
[root@linux-node2 qemu]# ls
instance-00000002.xml  networks
[root@linux-node2 qemu]# cd -
/var/lib/nova/instances/4e810c6c-5c56-4160-a29c-b240a4550679
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls
console.log  disk  disk.info  libvirt.xml
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# 

　　

只要用openstack管理，这个xml就会自动生成，openstack软重启不一定改变这个xml文件，但是硬重启一定会改变它
早期版本可以virsh手动改，以前是根据模板
openstack再加一层，变成类方式了。不能virsh更改xml文件了

在控制节点可以直接ssh此实例，而不用输入密码。
关于key是怎么跑进去的，通过metadata 元数据机制

[root@linux-node1 images]# ssh cirros@192.168.56.102
$ curl http://169.254.169.254/2009-04-04/meta-data
ami-id
ami-launch-index
ami-manifest-path
block-device-mapping/
hostname
instance-action
instance-id
instance-type
local-hostname
local-ipv4
placement/
public-hostname
public-ipv4
public-keys/
reservation-id

　　

获取ipv4的地址。 这就是它的meta-data

$ curl http://169.254.169.254/2009-04-04/meta-data/local-ipv4
192.168.56.102$ 
$ 

　　

这个169.254.169.254IP地址来自哪里呢

$ ip ad
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether fa:16:3e:ff:a2:6e brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.102/24 brd 192.168.56.255 scope global eth0
    inet6 fe80::f816:3eff:feff:a26e/64 scope link 
       valid_lft forever preferred_lft forever

有路由

$ ip ro li
default via 192.168.56.2 dev eth0 
169.254.169.254 via 192.168.56.100 dev eth0 
192.168.56.0/24 dev eth0  src 192.168.56.102 
$ 

　　

通过admin用户登录管理页面，找到了

dhcp在用它

登录计算节点去找它，下面命令没找到

[root@linux-node1 images]# ip ad li
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master brqac1b0655-93 state UP qlen 1000
    link/ether 00:0c:29:90:3c:7c brd ff:ff:ff:ff:ff:ff
3: virbr0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN 
    link/ether 52:54:00:fe:ab:0b brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
       valid_lft forever preferred_lft forever
4: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr0 state DOWN qlen 500
    link/ether 52:54:00:fe:ab:0b brd ff:ff:ff:ff:ff:ff
5: tap168a2111-98: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master brqac1b0655-93 state UP qlen 1000
    link/ether c2:5a:0d:4d:0d:88 brd ff:ff:ff:ff:ff:ff
6: brqac1b0655-93: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP 
    link/ether 00:0c:29:90:3c:7c brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.11/24 brd 192.168.56.255 scope global brqac1b0655-93
       valid_lft forever preferred_lft forever
[root@linux-node1 images]# 

　　

在namespace命名空间里找

[root@linux-node1 images]# ip netns li
qdhcp-ac1b0655-931d-4d6e-ba52-33fd0631e034
[root@linux-node1 images]# 

　　

在这个namespace里执行 ip ad li 看到了192.168.56.100

它还有个169.254.169.254

[root@linux-node1 images]# ip netns exec qdhcp-ac1b0655-931d-4d6e-ba52-33fd0631e034 ip ad li
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ns-168a2111-98: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether fa:16:3e:91:55:b6 brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.100/24 brd 192.168.56.255 scope global ns-168a2111-98
       valid_lft forever preferred_lft forever
    inet 169.254.169.254/16 brd 169.254.255.255 scope global ns-168a2111-98
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe91:55b6/64 scope link 
       valid_lft forever preferred_lft forever
[root@linux-node1 images]# 

　　

虚拟的这个路由来自于dhcp的推送,也是下面配置文件决定的

[root@linux-node1 ~]# grep enable_isolated  /etc/neutron/dhcp_agent.ini 
enable_isolated_metadata = True
# enable_isolated_metadata = True. (boolean value)
[root@linux-node1 ~]# 

　　 

你设置为true，它会追加主机路由

# The DHCP server can assist with providing metadata support on isolated
# networks. Setting this value to True will cause the DHCP server to append
# specific host routes to the DHCP request. The metadata service will only be
# activated when the subnet does not contain any router port. The guest
# instance must be configured to request host routes via DHCP (Option 121).
# This option doesn't have any effect when force_metadata is set to True.
# (boolean value)
enable_isolated_metadata = True

　　

这个等于true，你从控制节点才可以使用ssh-key登录
虚拟机的key就是自动从169.254.169.254这里获取的
这个命名空间里起了80端口的。之前curl访问元数据通的

[root@linux-node1 ~]# ip netns exec qdhcp-ac1b0655-931d-4d6e-ba52-33fd0631e034 lsof -i:80
COMMAND     PID    USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
neutron-n 25443 neutron    6u  IPv4  73251      0t0  TCP *:http (LISTEN)
[root@linux-node1 ~]# ip netns exec qdhcp-ac1b0655-931d-4d6e-ba52-33fd0631e034 netstat -lntp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
tcp        0      0 0.0.0.0:80              0.0.0.0:*               LISTEN      25443/python2       
tcp        0      0 192.168.56.100:53       0.0.0.0:*               LISTEN      25430/dnsmasq       
tcp        0      0 169.254.169.254:53      0.0.0.0:*               LISTEN      25430/dnsmasq       
tcp6       0      0 fe80::f816:3eff:fe91:53 :::*                    LISTEN      25430/dnsmasq       
[root@linux-node1 ~]# 

　　

它是通过下面方式获取key的

$ curl http://169.254.169.254/2009-04-04/meta-data/public-keys/0/openssh-key
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA+D1fhI9OjfYFl8Rt5nPPp8r1CfIUUh+RtqbarkBxcBHqP3gtG7dHt3lAt6i822X8YBMO5D3ch1p2SyIXToGPhLOdOC0eBxdmwvJitTCfA3ucAmTYa09HP/jlj
leLZ5Qx064AMLCVE/DRR8LlDX9dx49nNVYFUxQbGm6K7Ztx31FZN/O+o2mkb32OQ2y+D7aKtf62OloO8x27MxIM7X3YEVBZvmiszCWejFsX0m4/427iNT0vcGNOTT5oeAtNSH6SmO23ieg2SnIMDKulZ
m+Fl2Fx2uVP6R6m5IFD8U7TT6r5Xg0S91LDFcP8VjDCCwYZEFd+txer5fGZVVYR50v7 root@linux-node1.nmap.com
$ 

　　

也可以获取主机名

$ curl http://169.254.169.254/2009-04-04/meta-data/hostname
demo.novalocal$ 
$ 

　　

业内有个工具，也能帮你做此操作
方式一：cloud-init，帮你初始化，配置cloud-init挺麻烦
方式二：脚本方式也可以，更灵活

我们安装的这个小镜像带cloud-init，你自己做镜像，什么都不加，是无法连的。没法自动获取key的

知识点补充

1、怎么证明nova和glance没问题？执行nova image-list 列出结果，说明上面没问题
2、novncproxy如果挂掉，重启下很快的，没影响
3、600个虚拟机都可以使用单一扁平网络的，几千个虚拟机就不要用单一扁平网络了
4、一些公司现网中配置如下：2颗CPU 64GB内存，一个机器跑7个kvm
5、创建250个实例，30个物理机节点，单一扁平网络也是够的
6、16位的子网掩码，创建的更多
7、sdn（软件定义网络，比如ovs产品）的故障挺多，性能不好。移动架构师一次讲过不推荐
8、vlan最多是4096个，对于公有云远远不够。对于私有欲足够了
9、使用sdn的，主要是公有云
10、分布式路由早晚被干掉，只是把路由转到各个计算节点上
11、Openstack的数据库非常重要，数据库没了没法玩了
12、控制节点一般做高可用
13、单一扁平网络最稳定，中小型的公司足够了