qemu-system-aarch64 + telnet

https://mails.dpdk.org/archives/dts/2016-August/001780.html

 virt-sysprep -a  vhuser-test1.qcow2  --network --install "telnet*"

#qemu-system-aarch64  -name vm2 -nographic  
qemu-system-aarch64  -name vm2 -daemonize  
  -enable-kvm -M virt -cpu host -smp 2 -m 4096 
  -global virtio-blk-device.scsi=off 
  -device virtio-scsi-device,id=scsi 
  -kernel vmlinuz-4.18 --append "console=ttyAMA0  root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477" 
  -initrd initramfs-4.18 
 -drive file=vhuser-test1.qcow2  
 -netdev user,id=unet,hostfwd=tcp:127.0.0.1:1122-:22 -device virtio-net-device,netdev=unet 
 -monitor telnet::3333,server,nowait 
 -serial telnet:localhost:5432,server,nowait 
 -incoming tcp:0:4444 
 -vnc :10

Last login: Wed Nov  4 23:54:54 2020 from 192.168.117.59
[root@localhost ~]# telnet localhost 3333
Trying ::1...
Connected to localhost.
Escape character is '^]'.
QEMU 5.1.90 monitor - type 'help' for more information
(qemu) ls
unknown command: 'ls'
(qemu) info
info balloon  -- show balloon information
info block [-n] [-v] [device] -- show info of one block device or all block devices (-n: show named nodes; -v: show details)
info block-jobs  -- show progress of ongoing block device operations
info blockstats  -- show block device statistics
info capture  -- show capture information
info chardev  -- show the character devices
info cpus  -- show infos for each CPU
info cpustats  -- show CPU statistics
info dump  -- Display the latest dump status
info history  -- show the command line history
info hotpluggable-cpus  -- Show information about hotpluggable CPUs
info iothreads  -- show iothreads
info irq  -- show the interrupts statistics (if available)
info jit  -- show dynamic compiler info
info kvm  -- show KVM information
info memdev  -- show memory backends
info memory-devices  -- show memory devices
info memory_size_summary  -- show the amount of initially allocated and present hotpluggable (if enabled) memory in bytes.
info mice  -- show which guest mouse is receiving events
info migrate  -- show migration status
info migrate_cache_size  -- show current migration xbzrle cache size
info migrate_capabilities  -- show current migration capabilities
info migrate_parameters  -- show current migration parameters
info mtree [-f][-d][-o][-D] -- show memory tree (-f: dump flat view for address spaces;-d: dump dispatch tree, valid with -f only);-o: dump region owners/parents;-D: dump disabled regions
info name  -- show the current VM name
info network  -- show the network state
info numa  -- show NUMA information
info opcount  -- show dynamic compiler opcode counters
info pci  -- show PCI info
info pic  -- show PIC state
info profile  -- show profiling information
info qdm  -- show qdev device model list
info qom-tree [path] -- show QOM composition tree
info qtree  -- show device tree
info ramblock  -- Display system ramblock information
info rdma  -- show RDMA state
info registers [-a] -- show the cpu registers (-a: all - show register info for all cpus)
info replay  -- show record/replay information
info rocker name -- Show rocker switch
info rocker-of-dpa-flows name [tbl_id] -- Show rocker OF-DPA flow tables
info rocker-of-dpa-groups name [type] -- Show rocker OF-DPA groups
info rocker-ports name -- Show rocker ports
info roms  -- show roms
info snapshots  -- show the currently saved VM snapshots
info status  -- show the current VM status (running|paused)
info sync-profile [-m] [-n] [max] -- show synchronization profiling info, up to max entries (default: 10), sorted by total wait time. (-m: sort by mean wait time; -n: do not coalesce objects with the same call site)
info tpm  -- show the TPM device
info trace-events [name] [vcpu] -- show available trace-events & their state (name: event name pattern; vcpu: vCPU to query, default is any)
info usb  -- show guest USB devices
info usbhost  -- show host USB devices
info usernet  -- show user network stack connection states
info uuid  -- show the current VM UUID
info version  -- show the version of QEMU
info vm-generation-id  -- Show Virtual Machine Generation ID
info vnc  -- show the vnc server status
(qemu) info cpus
* CPU #0: thread_id=40195
  CPU #1: thread_id=40197
(qemu) info kvm
kvm support: enabled
(qemu) 

Configure script: configure.sh

git clone https://github.com/intel/nemu
cd nemu
git checkout -b topic/virt-x86 origin/topic/virt-x86
mkdir build-x86-64
cd build-x86-64
./configure.sh
make -j `nproc` 

Testing

We can test the NEMU virt machine type through either direct kernel boot (a.k.a. nofw) or through a regular cloud or server image as long as it supports EFI boot.

Direct kernel boot

Kernel

I use the v4.9.34-75 branch from the Clear Containers kernel with a custom .config

Initramfs image

See the busybox initramfs build instructions, and then configure the above kernel build to point to your local rootfs.

Command line

#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
# ex: ts=8 sw=4 sts=4 et filetype=sh


sudo /home/samuel/devlp/hypervisor/nemu/build-x86-64/x86_64-softmmu/qemu-system-x86_64 
     -nographic 
     -nodefaults 
     -L . 
     -net none 
     -machine virt,accel=kvm,kernel_irqchip,nofw 
     -smp sockets=1,cpus=4,cores=2,maxcpus=8 -cpu host 
     -m 1G,slots=3,maxmem=4G 
     -kernel /home/samuel/devlp/kernels/nfc/arch/x86/boot/compressed/vmlinux.bin -append 'console=hvc0 single iommu=false root=/dev/ram0' 
     -device virtio-serial-pci,id=virtio-serial0 -device virtconsole,chardev=charconsole0,id=console0 -chardev stdio,id=charconsole0 
     -monitor telnet:127.0.0.1:55555,server,nowait

Cloud image

Warning Most cloud or server images kernels do not have HW reduced ACPI enabled so they won't support CPU and memory hotplug with our HW reduced ACPI virt machine type.

We have tested the latest Clear Linux cloud image with our custom OVMF binary.

Command line

#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
# ex: ts=8 sw=4 sts=4 et filetype=sh

VMN=${VMN:=1}

sudo /home/samuel/devlp/hypervisor/nemu/build-x86-64/x86_64-softmmu/qemu-system-x86_64 
     -bios $HOME/devlp/hypervisor/nemu/clear-cloud/OVMF.fd 
     -nographic 
     -nodefaults 
     -L . 
     -machine virt,accel=kvm,kernel_irqchip 
     -smp sockets=1,cpus=4,cores=2,maxcpus=8 -cpu host 
     -m 512 
     -device sysbus-debugcon,iobase=0x402,chardev=debugcon -chardev file,path=/tmp/debug-log,id=debugcon 
     -device virtio-blk-pci,drive=image -drive if=none,id=image,file=$HOME/devlp/hypervisor/nemu/clear-cloud/clear-cloud.img 
     -device virtio-serial-pci,id=virtio-serial0 -device virtconsole,chardev=charconsole0,id=console0 -chardev stdio,id=charconsole0 
     -netdev user,id=mynet0,hostfwd=tcp::${VMN}0022-:22,hostfwd=tcp::${VMN}2375-:2375 -device virtio-net-pci,netdev=mynet0 
     -monitor telnet:127.0.0.1:55555,server,nowait

Hotplug

Hotplug is done through the QEMU monitoring socket (-monitor telnet:127.0.0.1:55555,server,nowait from the above command lines). The socket is reachable through port 23 (telnet):

# telnet 127.0.0.1 55555

CPU hotplug

Plug

(qemu) device_add host-x86_64-cpu,id=core4,socket-id=1,core-id=1,thread-id=0 

Unplug

(qemu) device_del core4

Memory hotplug

Plug

(qemu) object_add memory-backend-ram,id=mem1,size=1G
(qemu) device_add pc-dimm,id=dimm1,memdev=mem1

Unplug

(qemu) device_del dimm1

step

[root@localhost cloud_images]# vi sriov2.sh 
#qemu-system-aarch64  -name vm2 -daemonize 
qemu-system-aarch64  -name vm2 -nographic  
  -enable-kvm -M virt -cpu host -smp 2 -m 4096 
  -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on 
  -numa node,memdev=mem -mem-prealloc 
  -global virtio-blk-device.scsi=off 
  -device virtio-scsi-device,id=scsi 
  -kernel vmlinuz-4.18 --append "console=ttyAMA0  root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477 iommu=pt intel_iommu=on iommu.passthrough=1" 
  -initrd initramfs-4.18 
 -drive file=vhuser-test1.qcow2  
 -device vfio-pci,host=0000:05:00.0 
 -net nic,macaddr=00:00:00:99:99:01 
 -serial telnet:localhost:4321,server,nowait
 -vnc :10

qemu-system-aarch64  -name vm2 -daemonize 
  -enable-kvm -M virt -cpu host -smp 2 -m 4096 
  -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on 
  -numa node,memdev=mem -mem-prealloc 
  -global virtio-blk-device.scsi=off 
  -device virtio-scsi-device,id=scsi 
  -kernel vmlinuz-4.18 --append "console=ttyAMA0  root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477 iommu=pt intel_iommu=on iommu.passthrough=1" 
  -initrd initramfs-4.18 
 -drive file=vhuser-test1.qcow2  
 -device vfio-pci,host=0000:05:00.0 
 -net nic,macaddr=00:00:00:99:99:01 
 -serial telnet:localhost:4321,server,nowait
 -vnc :10

 step --host

[root@localhost cloud_images]# netstat -lpn | grep 4321
tcp        0      0 127.0.0.1:4321          0.0.0.0:*               LISTEN      46099/qemu-system-a 
tcp6       0      0 ::1:4321                :::*                    LISTEN      46099/qemu-system-a 
[root@localhost cloud_images]# telnet localhost 4321
Trying ::1...
Connected to localhost.
Escape character is '^]'.

[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:00:00:99:99:01 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::200:ff:fe99:9901/64 scope link 
       valid_lft forever preferred_lft forever
[root@localhost ~]# cat /proc/cmdline 
console=ttyAMA0  root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477 iommu=pt intel_iommu=on iommu.passthrough=1
[root@localhost ~]# lspci | grep -i ether
00:01.0 Ethernet controller: Red Hat, Inc. Virtio network device
00:02.0 Ethernet controller: Huawei Technologies Co., Ltd. Hi1822 Family (2*100GE) (rev 45)   ---透传的网卡不见了
[root@localhost ~]#