参考：https://ywnz.com/linuxjc/7269.html
yum install -y epel-release yum install -y centos-release-openstack-train yum install openvswitch libibverbs
[root@x86 ~]# systemctl enable --now openvswitch Created symlink /etc/systemd/system/multi-user.target.wants/openvswitch.service → /usr/lib/systemd/system/openvswitch.service. [root@x86 ~]# systemctl status openvswitch ● openvswitch.service - Open vSwitch Loaded: loaded (/usr/lib/systemd/system/openvswitch.service; enabled; vendor preset: disabled) Active: active (exited) since Thu 2020-07-02 08:22:38 UTC; 16s ago Process: 10852 ExecStart=/bin/true (code=exited, status=0/SUCCESS) Main PID: 10852 (code=exited, status=0/SUCCESS) Jul 02 08:22:38 x86.novalocal systemd[1]: Starting Open vSwitch... Jul 02 08:22:38 x86.novalocal systemd[1]: Started Open vSwitch. [root@x86 ~]# ovs-vsctl show e039e118-d51d-41de-8490-76e9bdc24e68 ovs_version: "2.12.0" [root@x86 ~]#
ovs-vsctl get-controller br-int
[root@x86 openvswitch-vtep]# ls /usr/share/openvswitch/scripts/ovs-vtep /usr/share/openvswitch/scripts/ovs-vtep [root@x86 openvswitch-vtep]#
git clone https://github.com/odivlad/openvswitch-vtep.git cd openvswitch-vtep sudo systemctl daemon-reload
[root@x86 ~]# cd openvswitch-vtep [root@x86 openvswitch-vtep]# ls ovs-vtepd.service README.md [root@x86 openvswitch-vtep]# cp ovs-vtepd.service /usr/lib/systemd/system [root@x86 openvswitch-vtep]#
[root@kunpeng82 openvswitch]# cat /usr/lib/systemd/system/ovs-vtepd.service [Unit] Description=Open vSwitch-based VTEP emulator After=openvswitch.service Requires=openvswitch.service [Service] Type=simple EnvironmentFile=/etc/sysconfig/openvswitch-vtep ExecStart=/usr/share/openvswitch/scripts/ovs-vtep --log-file=/var/log/openvswitch/ovs-vtep.log --pidfile=/var/run/openvswitch/ovs-vtep.pid $VTEP_SWITCH_NAME
[root@x86 openvswitch-vtep]# cp ovs-vtepd.service /usr/lib/systemd/system [root@x86 openvswitch-vtep]# systemctl daemon-reload [root@x86 openvswitch-vtep]# ls /etc/sysconfig/openvswitch /etc/sysconfig/openvswitch [root@x86 openvswitch-vtep]# echo 'OPTIONS=${OPTIONS} --extra-dbs=hw_vtep.db' >> /etc/sysconfig/openvswitch [root@x86 openvswitch-vtep]# ovsdb-tool create /etc/openvswitch/hw_vtep.db /usr/share/openvswitch/vtep.ovsschema [root@x86 openvswitch-vtep]# systemctl restart openvswitch A dependency job for openvswitch.service failed. See 'journalctl -xe' for details. [root@x86 openvswitch-vtep]# systemctl status openvswitch ● openvswitch.service - Open vSwitch Loaded: loaded (/usr/lib/systemd/system/openvswitch.service; enabled; vendor preset: disabled) Active: inactive (dead) since Thu 2020-07-02 08:30:11 UTC; 18s ago Process: 13995 ExecStop=/bin/true (code=exited, status=0/SUCCESS) Main PID: 10852 (code=exited, status=0/SUCCESS) Jul 02 08:22:38 x86.novalocal systemd[1]: Starting Open vSwitch... Jul 02 08:22:38 x86.novalocal systemd[1]: Started Open vSwitch. Jul 02 08:30:11 x86.novalocal systemd[1]: Stopping Open vSwitch... Jul 02 08:30:11 x86.novalocal systemd[1]: Stopped Open vSwitch. Jul 02 08:30:11 x86.novalocal systemd[1]: Dependency failed for Open vSwitch. Jul 02 08:30:11 x86.novalocal systemd[1]: openvswitch.service: Job openvswitch.service/start failed with result '> [root@x86 openvswitch-vtep]#
[root@x86 openvswitch-vtep]# journalctl -xe -- Subject: Unit ovsdb-server.service has failed -- Defined-By: systemd -- Support: https://access.redhat.com/support -- -- Unit ovsdb-server.service has failed. -- -- The result is RESULT. Jul 02 08:30:13 x86.novalocal systemd[1]: ovsdb-server.service: Service RestartSec=100ms expired, scheduling rest> Jul 02 08:30:13 x86.novalocal systemd[1]: ovsdb-server.service: Scheduled restart job, restart counter is at 5. -- Subject: Automatic restarting of a unit has been scheduled -- Defined-By: systemd -- Support: https://access.redhat.com/support -- -- Automatic restarting of the unit ovsdb-server.service has been scheduled, as the result for -- the configured Restart= setting for the unit. Jul 02 08:30:13 x86.novalocal systemd[1]: Stopped Open vSwitch Database Unit. -- Subject: Unit ovsdb-server.service has finished shutting down -- Defined-By: systemd -- Support: https://access.redhat.com/support -- -- Unit ovsdb-server.service has finished shutting down. Jul 02 08:30:13 x86.novalocal systemd[1]: ovsdb-server.service: Start request repeated too quickly. Jul 02 08:30:13 x86.novalocal systemd[1]: ovsdb-server.service: Failed with result 'exit-code'. Jul 02 08:30:13 x86.novalocal systemd[1]: Failed to start Open vSwitch Database Unit. -- Subject: Unit ovsdb-server.service has failed -- Defined-By: systemd -- Support: https://access.redhat.com/support -- -- Unit ovsdb-server.service has failed. -- -- The result is RESULT. [root@x86 openvswitch-vtep]#
看这个https://bugzilla.redhat.com/show_bug.cgi?id=1765461发现是系统版本的问题
换个系统
[root@kunpeng82 openvswitch-vtep]# uname -a Linux kunpeng82.bogon 4.18.0-147.8.1.el7.aarch64 #1 SMP Wed Apr 15 18:13:44 UTC 2020 aarch64 aarch64 aarch64 GNU/Linux
[devuser@kunpeng82 openvswitch-vtep]$ cp ovs-vtepd.service /usr/lib/systemd/system cp: cannot create regular file ‘/usr/lib/systemd/system/ovs-vtepd.service’: Permission denied [devuser@kunpeng82 openvswitch-vtep]$ sudo su [root@kunpeng82 openvswitch-vtep]# cp ovs-vtepd.service /usr/lib/systemd/system [root@kunpeng82 openvswitch-vtep]# systemctl daemon-reload
[root@kunpeng82 ~]# touch /etc/sysconfig/openvswitch-vtep
[root@kunpeng82 ~]# echo 'OPTIONS=${OPTIONS} --extra-dbs=hw_vtep.db' >> /etc/sysconfig/openvswitch-vtep
[root@kunpeng82 openvswitch-vtep]# echo 'OPTIONS=${OPTIONS} --extra-dbs=hw_vtep.db' >> /etc/sysconfig/openvswitch [root@kunpeng82 openvswitch-vtep]# ovsdb-tool create /etc/openvswitch/hw_vtep.db /usr/share/openvswitch/vtep.ovsschema [root@kunpeng82 openvswitch-vtep]# systemctl restart openvswitch [root@kunpeng82 openvswitch-vtep]# unam -a bash: unam: command not found [root@kunpeng82 openvswitch-vtep]# uname -a Linux kunpeng82.bogon 4.18.0-147.8.1.el7.aarch64 #1 SMP Wed Apr 15 18:13:44 UTC 2020 aarch64 aarch64 aarch64 GNU/Linux [root@kunpeng82 openvswitch-vtep]# systemctl status openvswitch ● openvswitch.service - LSB: Open vSwitch switch Loaded: loaded (/etc/rc.d/init.d/openvswitch; bad; vendor preset: disabled) Active: active (running) since Thu 2020-07-02 08:41:03 UTC; 24s ago Docs: man:systemd-sysv-generator(8) Process: 5126 ExecStop=/etc/rc.d/init.d/openvswitch stop (code=exited, status=0/SUCCESS) Process: 5546 ExecStart=/etc/rc.d/init.d/openvswitch start (code=exited, status=0/SUCCESS) CGroup: /system.slice/openvswitch.service ├─5572 ovsdb-server: monitoring pid 5573 (healthy) ├─5573 ovsdb-server /etc/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:... ├─5588 ovs-vswitchd: monitoring pid 5589 (healthy) └─5589 ovs-vswitchd unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mloc... Jul 02 08:40:57 kunpeng82.bogon ovs-vsctl[5574]: ovs|00001|vsctl|INFO|Called as ovs-vsctl --no-wait -- init....0.0 Jul 02 08:40:57 kunpeng82.bogon ovs-vsctl[5579]: ovs|00001|vsctl|INFO|Called as ovs-vsctl --no-wait set Ope...7"" Jul 02 08:40:57 kunpeng82.bogon openvswitch[5546]: Configuring Open vSwitch system IDs [ OK ] Jul 02 08:40:57 kunpeng82.bogon ovs-vswitchd[5589]: ovs|00040|socket_util|ERR|6654: bind: Permission denied Jul 02 08:40:57 kunpeng82.bogon ovs-vswitchd[5589]: ovs|00048|socket_util|ERR|6656: bind: Permission denied Jul 02 08:40:57 kunpeng82.bogon ovs-vswitchd[5589]: ovs|00050|socket_util|ERR|6655: bind: Permission denied Jul 02 08:40:57 kunpeng82.bogon openvswitch[5546]: Starting ovs-vswitchd [ OK ] Jul 02 08:40:57 kunpeng82.bogon openvswitch[5546]: Enabling remote OVSDB managers [ OK ] Jul 02 08:41:03 kunpeng82.bogon ovs-vsctl[5863]: ovs|00001|vsctl|INFO|Called as ovs-vsctl --no-wait set Ope...ogon Jul 02 08:41:03 kunpeng82.bogon systemd[1]: Started LSB: Open vSwitch switch. Hint: Some lines were ellipsized, use -l to show in full. [root@kunpeng82 openvswitch-vtep]#
[root@kunpeng82 openvswitch-vtep]# ovs-vsctl show ec7f002f-352c-476c-8348-5f2c86c8194f ovs_version: "2.12.0" [root@kunpeng82 openvswitch-vtep]#
[root@kunpeng82 ~]# ovs-vsctl add-br br0 [root@kunpeng82 ~]# ovs-vsctl add-port br0 enahisic2i3 [root@kunpeng82 ~]# vtep-ctl add-ps br0 vtep-ctl: unix:/var/run/openvswitch/db.sock: database connection failed () [root@kunpeng82 ~]#
[root@kunpeng82 ~]# ls /etc/openvswitch/
conf.db conf.db.bak hw_vtep.db system-id.conf
[root@kunpeng82 ~]# systemctl status ovsdb-server Unit ovsdb-server.service could not be found. [root@kunpeng82 ~]# ps -elf | grep ovsdb 1 S root 6448 1 0 70 -10 - 270 do_wai 08:48 ? 00:00:00 ovsdb-server: monitoring pid 6449 (healthy) 1 S root 6449 6448 0 70 -10 - 275 do_sys 08:48 ? 00:00:00 ovsdb-server /etc/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/var/run/openvswitch/db.sock --private-key=db:Open_vSwitch,SSL,private_key --certificate=db:Open_vSwitch,SSL,certificate --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --no-chdir --log-file=/var/log/openvswitch/ovsdb-server.log --pidfile=/var/run/openvswitch/ovsdb-server.pid --detach --monitor 0 S root 6692 5082 0 80 0 - 1730 pipe_w 09:23 pts/1 00:00:00 grep --color=auto ovsdb [root@kunpeng82 ~]#
重新启动ovsdb
[root@kunpeng82 ~]# ps -elf | grep ovsdb 1 S root 6448 1 0 70 -10 - 270 do_wai 08:48 ? 00:00:00 ovsdb-server: monitoring pid 6449 (healthy) 1 S root 6449 6448 0 70 -10 - 275 do_sys 08:48 ? 00:00:00 ovsdb-server /etc/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/var/run/openvswitch/db.sock --private-key=db:Open_vSwitch,SSL,private_key --certificate=db:Open_vSwitch,SSL,certificate --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --no-chdir --log-file=/var/log/openvswitch/ovsdb-server.log --pidfile=/var/run/openvswitch/ovsdb-server.pid --detach --monitor 0 S root 6716 5082 0 80 0 - 1730 pipe_w 09:33 pts/1 00:00:00 grep --color=auto ovsdb [root@kunpeng82 ~]# kill -9 6449 [root@kunpeng82 ~]# ps -elf | grep ovsdb 0 S root 6720 5082 0 80 0 - 1729 pipe_w 09:33 pts/1 00:00:00 grep --color=auto ovsdb [root@kunpeng82 ~]# ps -elf | grep ovsdb 0 S root 6724 5082 0 80 0 - 1729 pipe_w 09:33 pts/1 00:00:00 grep --color=auto ovsdb [root@kunpeng82 ~]# ps -elf | grep ovsdb
[root@kunpeng82 ~]# ps -elf | grep ovsdb 1 S root 6731 1 0 80 0 - 270 do_wai 09:37 ? 00:00:00 ovsdb-server: monitoring pid 6732 (healthy) 1 S root 6732 6731 0 80 0 - 279 do_sys 09:37 ? 00:00:00 ovsdb-server /etc/openvswitch/conf.db /etc/openvswitch/hw_vtep.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/var/run/openvswitch/db.sock --remote=db:hardware_vtep,Global,managers --private-key=db:Open_vSwitch,SSL,private_key --certificate=db:Open_vSwitch,SSL,certificate --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --no-chdir --log-file=/var/log/openvswitch/ovsdb-server.log --pidfile=/var/run/openvswitch/ovsdb-server.pid --detach --monitor 0 S root 6734 5082 0 80 0 - 1730 pipe_w 09:38 pts/1 00:00:00 grep --color=auto ovsdb [root@kunpeng82 ~]# vtep-ctl add-ps br0 -----------成功了 [root@kunpeng82 ~]#
[root@kunpeng82 ~]# vtep-ctl add-ps br0 [root@kunpeng82 ~]# vtep-ctl set Physical_Switch br0 tunnel_ips=10.2.2.1 [root@kunpeng82 ~]#
[root@kunpeng82 ~]# echo 'OPTIONS=${OPTIONS} --extra-dbs=hw_vtep.db' >> /etc/sysconfig/openvswitch-vtep [root@kunpeng82 ~]# systemctl start ovs-vtepd [root@kunpeng82 ~]# systemctl status ovs-vtepd ● ovs-vtepd.service - Open vSwitch-based VTEP emulator Loaded: loaded (/usr/lib/systemd/system/ovs-vtepd.service; static; vendor preset: disabled) Active: failed (Result: exit-code) since Thu 2020-07-02 09:59:22 UTC; 46s ago Process: 6789 ExecStart=/usr/share/openvswitch/scripts/ovs-vtep --log-file=/var/log/openvswitch/ovs-vtep.log --pidfile=/var/run/openvswitch/ovs-vtep.pid $VTEP_SWITCH_NAME (code=exited, status=1/FAILURE) Main PID: 6789 (code=exited, status=1/FAILURE) Jul 02 09:59:22 kunpeng82.bogon systemd[1]: Started Open vSwitch-based VTEP emulator. Jul 02 09:59:22 kunpeng82.bogon ovs-vtep[6789]: Traceback (most recent call last): Jul 02 09:59:22 kunpeng82.bogon ovs-vtep[6789]: File "/usr/share/openvswitch/scripts/ovs-vtep", line 26, in...ule> Jul 02 09:59:22 kunpeng82.bogon ovs-vtep[6789]: import ovs.daemon Jul 02 09:59:22 kunpeng82.bogon ovs-vtep[6789]: ImportError: No module named ovs.daemon Jul 02 09:59:22 kunpeng82.bogon systemd[1]: ovs-vtepd.service: main process exited, code=exited, status=1/FAILURE Jul 02 09:59:22 kunpeng82.bogon systemd[1]: Unit ovs-vtepd.service entered failed state. Jul 02 09:59:22 kunpeng82.bogon systemd[1]: ovs-vtepd.service failed. Hint: Some lines were ellipsized, use -l to show in full.
[root@kunpeng82 ovs]# ls
compat dirs.py __init__.py ovsuuid.py __pycache__ stream.py util.py winutils.py
daemon.py fatal_signal.py json.py poller.py reconnect.py timeval.py version.py
db fcntl_win.py jsonrpc.py process.py socket_util.py unixctl vlog.py
[root@kunpeng82 ovs]#
[root@kunpeng82 ovs]# python -V Python 2.7.5 [root@kunpeng82 ovs]# pip -V bash: pip: command not found [root@kunpeng82 ovs]# yum -y install python-pip
[root@kunpeng82 ovs]# pip install ovs
[root@kunpeng82 ~]# /usr/share/openvswitch/scripts/ovs-vtep --log-file=/var/log/openvswitch/ovs-vtep.log > --pidfile=/var/run/openvswitch/ovs-vtep.pid > --detach br0 [root@kunpeng82 ~]# ps -elf | grep ovs-vtep 4 S root 6987 6742 0 80 0 - 1779 do_wai 10:12 pts/2 00:00:00 systemctl show ovs-vtepd.service 1 S root 6990 1 0 80 0 - 2049 pipe_w 10:12 ? 00:00:00 /bin/python2 /usr/share/openvswitch/scripts/ovs-vtep --log-file=/var/log/openvswitch/ovs-vtep.log --pidfile=/var/run/openvswitch/ovs-vtep.pid --detach br0 0 S root 7000 5082 0 80 0 - 1729 pipe_w 10:12 pts/1 00:00:00 grep --color=auto ovs-vtep [root@kunpeng82 ~]#
[root@kunpeng82 ~]# ovs-vsctl show ec7f002f-352c-476c-8348-5f2c86c8194f Bridge vtep_bfd Port vtep_bfd Interface vtep_bfd type: internal Bridge "br0" Port "enahisic2i2" Interface "enahisic2i2" Port "br0" Interface "br0" type: internal ovs_version: "2.12.0" [root@kunpeng82 ~]# ip a | grep vtep_bfd 246: vtep_bfd: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 [root@kunpeng82 ~]# ip a s how vtep_bfd Error: either "dev" is duplicate, or "vtep_bfd" is a garbage. [root@kunpeng82 ~]# ip a sh vtep_bfd 246: vtep_bfd: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 link/ether 62:a3:ad:09:0b:4c brd ff:ff:ff:ff:ff:ff [root@kunpeng82 ~]#
[root@kunpeng82 ~]# vtep-ctl show 7ad71c45-a370-4718-89cd-c15d142f1fce Physical_Switch "br0" tunnel_ips: ["10.2.2.1"] Physical_Port "enahisic2i2" vlan_bindings: [root@kunpeng82 ~]#
ip link set up dev ens5 apt-get install openvswitch-vtep ovsdb-tool create /etc/openvswitch/vtep.db /usr/share/openvswitch/vtep.ovsschema ovsdb-tool create /etc/openvswitch/vswitch.db /usr/share/openvswitch/vswitch.ovsschema ovsdb-server --pidfile --detach --log-file --remote ptcp:6632:10.225.0.27 --remote punix:/var/run/openvswitch/db.sock --remote=db:hardware_vtep,Global,managers /etc/openvswitch/vswitch.db /etc/openvswitch/vtep.db ovs-vswitchd --log-file --detach --pidfile unix:/var/run/openvswitch/db.sock ovs-vsctl add-br myphyswitch vtep-ctl add-ps myphyswitch vtep-ctl set Physical_Switch myphyswitch tunnel_ips=10.225.0.27 ovs-vsctl add-port myphyswitch ens5 vtep-ctl add-port myphyswitch ens5 /usr/share/openvswitch/scripts/ovs-vtep --log-file=/var/log/openvswitch/ovs-vtep.log --pidfile=/var/run/openvswitch/ovs-vtep.pid --detach myphyswitch
https://tungsten.io/opencontrail-sdn-lab-testing-1-tor-switches-with-ovsdb/
https://docs.midonet.org/docs/latest-en/operations-guide/content/vxgw_troubleshooting.html
[root@kunpeng82 ~]# ovsdb-client list-dbs unix:/var/run/openvswitch/db.sock Open_vSwitch _Server hardware_vtep [root@kunpeng82 ~]# vtep-ctl list Physical_Switch _uuid : 7a04316a-1e20-4bb6-948d-6512c93ed30d description : "OVS VTEP Emulator" management_ips : [] name : "br0" other_config : {} ports : [7a9c6e51-fc3f-4725-88d9-ddd94cae91ed] switch_fault_status : [] tunnel_ips : ["10.2.2.1"] tunnels : [] [root@kunpeng82 ~]#
#ip netns add ns1 #ip link add nstap1 type veth peer name tortap1 #ovs-vsctl add-port br0 tortap1 #ip link set nstap1 netns ns1 #ip netns exec ns1 ip link set dev nstap1 up #ip link set dev tortap1 up ip netns exec ns1 ip a a 127.0.0.1/8 dev lo ip netns exec ns1 ip a ip netns exec ns1 ip a a 10.0.10.120/24 dev nstap1 ip netns exec ns1 ping 10.0.10.120 ip netns exec ns1 ip link set up dev lo ip netns exec ns1 ping 10.0.10.120
[root@kunpeng82 ~]# vtep-ctl list Physical_Switch _uuid : 7a04316a-1e20-4bb6-948d-6512c93ed30d description : "OVS VTEP Emulator" management_ips : [] name : "br0" other_config : {} ports : [50d5083a-9b0d-42bb-a186-6fcf7a06159a, 7a9c6e51-fc3f-4725-88d9-ddd94cae91ed] switch_fault_status : [] tunnel_ips : ["10.2.2.1"] tunnels : [] [root@kunpeng82 ~]# vsdb-client list-dbs unix:/var/run/openvswitch/db.sock bash: vsdb-client: command not found [root@kunpeng82 ~]# ovsdb-client list-dbs unix:/var/run/openvswitch/db.sock Open_vSwitch _Server hardware_vtep [root@kunpeng82 ~]#
[root@kunpeng82 ~]# vtep-ctl list-ls
[root@kunpeng82 ~]#
[root@kunpeng82 ~]# vtep-ctl list-ls [root@kunpeng82 ~]# vtep-ctl show 7ad71c45-a370-4718-89cd-c15d142f1fce Physical_Switch "br0" tunnel_ips: ["10.2.2.1"] Physical_Port "tortap1" vlan_bindings: Physical_Port "enahisic2i2" vlan_bindings: [root@kunpeng82 ~]#
[root@kunpeng82 ~]# ovsdb-client dump hardware_vtep ACL table _uuid acl_entries acl_fault_status acl_name ----- ----------- ---------------- -------- ACL_entry table _uuid acle_fault_status action dest_ip dest_mac dest_mask dest_port_max dest_port_min direction ethertype icmp_code icmp_type protocol sequence source_ip source_mac source_mask source_port_max source_port_min tcp_flags tcp_flags_mask ----- ----------------- ------ ------- -------- --------- ------------- ------------- --------- --------- --------- --------- -------- -------- --------- ---------- ----------- --------------- --------------- --------- -------------- Arp_Sources_Local table _uuid locator src_mac ----- ------- ------- Arp_Sources_Remote table _uuid locator src_mac ----- ------- ------- Global table _uuid managers other_config switches ------------------------------------ -------- ------------ -------------------------------------- 7ad71c45-a370-4718-89cd-c15d142f1fce [] {} [7a04316a-1e20-4bb6-948d-6512c93ed30d] Logical_Binding_Stats table _uuid bytes_from_local bytes_to_local packets_from_local packets_to_local ----- ---------------- -------------- ------------------ ---------------- Logical_Router table LR_fault_status _uuid acl_binding description name other_config static_routes switch_binding --------------- ----- ----------- ----------- ---- ------------ ------------- -------------- Logical_Switch table _uuid description name other_config replication_mode tunnel_key ----- ----------- ---- ------------ ---------------- ---------- Manager table _uuid inactivity_probe is_connected max_backoff other_config status target ----- ---------------- ------------ ----------- ------------ ------ ------ Mcast_Macs_Local table MAC _uuid ipaddr locator_set logical_switch --- ----- ------ ----------- -------------- Mcast_Macs_Remote table MAC _uuid ipaddr locator_set logical_switch --- ----- ------ ----------- -------------- Physical_Locator table _uuid dst_ip encapsulation_type tunnel_key ----- ------ ------------------ ---------- Physical_Locator_Set table _uuid locators ----- -------- Physical_Port table _uuid acl_bindings description name other_config port_fault_status vlan_bindings vlan_stats ------------------------------------ ------------ ----------- ------------- ------------ ----------------- ------------- ---------- 7a9c6e51-fc3f-4725-88d9-ddd94cae91ed {} "" "enahisic2i2" {} [] {} {} 50d5083a-9b0d-42bb-a186-6fcf7a06159a {} "" "tortap1" {} [] {} {} Physical_Switch table _uuid description management_ips name other_config ports switch_fault_status tunnel_ips tunnels ------------------------------------ ------------------- -------------- ----- ------------ ---------------------------------------------------------------------------- ------------------- ------------ ------- 7a04316a-1e20-4bb6-948d-6512c93ed30d "OVS VTEP Emulator" [] "br0" {} [50d5083a-9b0d-42bb-a186-6fcf7a06159a, 7a9c6e51-fc3f-4725-88d9-ddd94cae91ed] [] ["10.2.2.1"] [] Tunnel table _uuid bfd_config_local bfd_config_remote bfd_params bfd_status local remote ----- ---------------- ----------------- ---------- ---------- ----- ------ Ucast_Macs_Local table MAC _uuid ipaddr locator logical_switch --- ----- ------ ------- -------------- Ucast_Macs_Remote table MAC _uuid ipaddr locator logical_switch --- ----- ------ ------- -------------- [root@kunpeng82 ~]#
[root@kunpeng82 ~]# vtep-ctl get-manager [root@kunpeng82 ~]# ovs-vsctl get-controller br0 [root@kunpeng82 ~]# vtep-ctl list-remote-macs br0 vtep-ctl: no logical switch named br0 [root@kunpeng82 ~]#
[root@kunpeng82 ~]# vtep-ctl add-ls ls0 [root@kunpeng82 ~]# vtep-ctl list-remote-macs ls0 ucast-mac-remote mcast-mac-remote [root@kunpeng82 ~]#
[root@kunpeng82 ~]# vtep-ctl bind-ls br0 p0 0 ls0 vtep-ctl: no port named p0 [root@kunpeng82 ~]#
不能对tortap1再ovs tcpdump
[root@kunpeng82 ~]# vtep-ctl set Logical_Switch ls0 tunnel_key=33 [root@kunpeng82 ~]#
[root@kunpeng82 ~]# vtep-ctl set Logical_Switch ls0 tunnel_key=33 [root@kunpeng82 ~]# ip link add vxlan33 type vxlan id 33 remote dstport 4789 dev enahisic2i2 Invalid address "dstport" [root@kunpeng82 ~]# ip link add vxlan33 type vxlan id 33 dstport 4789 dev enahisic2i2 [root@kunpeng82 ~]# ip a add 10.2.2.1/24 dev enahisic2i2 [root@kunpeng82 ~]#
[root@kunpeng82 ~]#
[root@kunpeng82 ~]# ip link set vxlan33 up RTNETLINK answers: Address already in use
[root@kunpeng82 ~]# ip a add 10.2.2.1/24 dev enahisic2i2 [root@kunpeng82 ~]# vtep-ctl list-remote-macs ls0 ucast-mac-remote mcast-mac-remote unknown-dst -> vxlan_over_ipv4/10.2.2.2 [root@kunpeng82 ~]#
另外一个节点
[root@bogon ~]# ip link add vxlan33 type vxlan id 33 dstport 4789 dev enahisic2i2 [root@bogon ~]# ip a add 10.2.2.2/24 dev enahisic2i2 [root@bogon ~]# vi tap.sh 1 ip netns add ns1 2 ip link add nstap1 type veth peer name tortap1 3 ip link set nstap1 netns ns1 4 ip netns exec ns1 ip link set dev nstap1 up 5 ip link set dev tortap1 up 6 7 ip netns exec ns1 ip a a 127.0.0.1/8 dev lo 8 ip netns exec ns1 ip a 9 ip netns exec ns1 ip a a 10.0.10.120/24 dev nstap1 10 ip netns exec ns1 ping 10.0.10.120 11 ip netns exec ns1 ip link set up dev lo 12 ip netns exec ns1 ping 10.0.10.120 ~
brctl addif br0 vxlan33
[root@bogon ~]# ip a flush enahisic2i2 [root@bogon ~]# ip a add 10.2.2.2/24 dev enahisic2i2 [root@bogon ~]# ip link delete vxlan33 [root@bogon ~]# ip link add vxlan33 type vxlan id 33 remote 10.2.2.1/24 dstport 4789 dev vxlan33 Invalid address "10.2.2.1/24" [root@bogon ~]# ip link add vxlan33 type vxlan id 33 remote 10.2.2.1 dstport 4789 dev vxlan33 Cannot find device "vxlan33" [root@bogon ~]# ip link add vxlan33 type vxlan id 33 remote 10.2.2.1 dstport 4789 dev enahisic2i2 [root@bogon ~]#
[root@kunpeng82 devuser]# ip netns exec ns1 ping 10.0.10.121 PING 10.0.10.121 (10.0.10.121) 56(84) bytes of data. From 10.0.10.120 icmp_seq=1 Destination Host Unreachable From 10.0.10.120 icmp_seq=2 Destination Host Unreachable From 10.0.10.120 icmp_seq=3 Destination Host Unreachable From 10.0.10.120 icmp_seq=4 Destination Host Unreachable From 10.0.10.120 icmp_seq=5 Destination Host Unreachable From 10.0.10.120 icmp_seq=6 Destination Host Unreachable ^C --- 10.0.10.121 ping statistics --- 7 packets transmitted, 0 received, +6 errors, 100% packet loss, time 6234ms pipe 4 [root@kunpeng82 devuser]# ip netns exec ns1 ping 10.10.10.121 connect: Network is unreachable [root@kunpeng82 devuser]#
更改tunnel ip
[root@kunpeng82 devuser]# vtep-ctl show 7ad71c45-a370-4718-89cd-c15d142f1fce Physical_Switch "br0" tunnel_ips: ["10.2.2.1"] Physical_Port "tortap1" vlan_bindings: 0="ls0" [root@kunpeng82 devuser]# vtep-ctl set Physical_Switch br0 tunnel_ips=192.168.16.81 [root@kunpeng82 devuser]# vtep-ctl show 7ad71c45-a370-4718-89cd-c15d142f1fce Physical_Switch "br0" tunnel_ips: ["192.168.16.81"] Physical_Port "tortap1" vlan_bindings: 0="ls0" [root@kunpeng82 devuser]#
[root@kunpeng82 devuser]# ovs-vsctl del-port br0 enahisic2i2 [root@kunpeng82 devuser]# ip netns exec ns1 ping 10.10.10.121 connect: Network is unreachable [root@kunpeng82 devuser]# ip a flush enahisic2i2 [root@kunpeng82 devuser]# ip a add 192.168.16.82/24 dev enahisic2i2
[root@kunpeng82 devuser]# vtep-ctl add-mcast-remote ls0 unknown-dst 192.168.16.81 [root@kunpeng82 devuser]# vtep-ctl list-remote-macs ls0 ucast-mac-remote mcast-mac-remote unknown-dst -> vxlan_over_ipv4/10.2.2.2 unknown-dst -> vxlan_over_ipv4/192.168.16.81 [root@kunpeng82 devuser]# vtep-ctl del-mcast-remote ls0 unknown-dst 10.2.2.2 [root@kunpeng82 devuser]# vtep-ctl list-remote-macs ls0 ucast-mac-remote mcast-mac-remote unknown-dst -> vxlan_over_ipv4/192.168.16.81 [root@kunpeng82 devuser]#
[root@bogon ~]# ip netns exec ns1 ping 10.0.10.120 PING 10.0.10.120 (10.0.10.120) 56(84) bytes of data. 64 bytes from 10.0.10.120: icmp_seq=1 ttl=64 time=1.06 ms 64 bytes from 10.0.10.120: icmp_seq=2 ttl=64 time=0.154 ms 64 bytes from 10.0.10.120: icmp_seq=3 ttl=64 time=0.107 ms 64 bytes from 10.0.10.120: icmp_seq=4 ttl=64 time=0.129 ms ^C --- 10.0.10.120 ping statistics --- 4 packets transmitted, 4 received, 0% packet loss, time 3047ms rtt min/avg/max/mdev = 0.107/0.363/1.065/0.405 ms [root@bogon ~]#
[root@kunpeng82 devuser]# ip netns exec ns1 ping 10.0.10.121 PING 10.0.10.121 (10.0.10.121) 56(84) bytes of data. 64 bytes from 10.0.10.121: icmp_seq=1 ttl=64 time=0.659 ms 64 bytes from 10.0.10.121: icmp_seq=2 ttl=64 time=0.169 ms ^C --- 10.0.10.121 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1058ms rtt min/avg/max/mdev = 0.169/0.414/0.659/0.245 ms [root@kunpeng82 devuser]#
81节点
ip link add vxlan33 type vxlan id 33 remote 192.168.16.82 dstport 4789 dev enahisic2i2 ip link set vxlan33 up
[root@kunpeng82 devuser]# vtep-ctl list-ls ls0 [root@kunpeng82 devuser]# vtep-ctl list-ls list-remote-macs ls0 vtep-ctl: 'list-ls' command takes at most 0 arguments [root@kunpeng82 devuser]# vtep-ctl list-remote-macs ls0 ucast-mac-remote mcast-mac-remote unknown-dst -> vxlan_over_ipv4/192.168.16.81 [root@kunpeng82 devuser]#
[root@kunpeng82 devuser]# vtep-ctl list Physical_Port _uuid : 50d5083a-9b0d-42bb-a186-6fcf7a06159a acl_bindings : {} description : "" name : "tortap1" other_config : {} port_fault_status : [] vlan_bindings : {0=bb0b536c-be0c-400f-9a77-9740d58b9625} vlan_stats : {0=1b427e5d-fe0d-4a4c-9c6e-8f575eb84acf} [root@kunpeng82 devuser]#
[root@kunpeng82 devuser]# ps -elf | grep ovs 1 S root 6731 1 0 80 0 - 270 do_wai Jul02 ? 00:00:00 ovsdb-server: monitoring pid 6732 (healthy) 1 S root 6732 6731 2 80 0 - 279 do_sys Jul02 ? 00:30:53 ovsdb-server /etc/openvswitch/conf.db /etc/openvswitch/hw_vtep.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/var/run/openvswitch/db.sock --remote=db:hardware_vtep,Global,managers --private-key=db:Open_vSwitch,SSL,private_key --certificate=db:Open_vSwitch,SSL,certificate --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --no-chdir --log-file=/var/log/openvswitch/ovsdb-server.log --pidfile=/var/run/openvswitch/ovsdb-server.pid --detach --monitor 5 S root 7333 1 1 80 0 - 75246 do_sys Jul02 ? 00:17:27 ovs-vswitchd --log-file --detach --pidfile unix:/var/run/openvswitch/db.sock 1 S root 11818 1 4 80 0 - 2053 do_sel Jul02 ? 00:42:33 /bin/python2 /usr/share/openvswitch/scripts/ovs-vtep --log-file=/var/log/openvswitch/ovs-vtep.log --pidfile=/var/run/openvswitch/ovs-vtep.pid --detach br0 4 S root 19898 20712 0 80 0 - 1730 pipe_w 03:26 pts/0 00:00:00 grep --color=auto ovs [root@kunpeng82 devuser]# vsdb-client list-tables unix:/var/run/openvswitch/db.sock hardware_vtep bash: vsdb-client: command not found [root@kunpeng82 devuser]# ovsdb-client list-tables unix:/var/run/openvswitch/db.sock hardware_vtep Table --------------------- Physical_Locator_Set Mcast_Macs_Remote Physical_Port Global Logical_Switch ACL Arp_Sources_Remote Physical_Locator Ucast_Macs_Remote Mcast_Macs_Local Ucast_Macs_Local Arp_Sources_Local Logical_Binding_Stats Manager Logical_Router ACL_entry Physical_Switch Tunnel [root@kunpeng82 devuser]#
删除 unknown-dst
[root@kunpeng82 devuser]# vtep-ctl add-mcast-remote ls0 unknown-dst 192.168.16.81 [root@kunpeng82 devuser]# vtep-ctl del-mcast-remote ls0 unknown-dst 192.168.16.81 [root@kunpeng82 devuser]# vtep-ctl list-remote-macs ls0 ucast-mac-remote mcast-mac-remote [root@kunpeng82 devuser]#
无法ping通
[root@bogon ~]# ip netns exec ns1 ping 10.0.10.120 PING 10.0.10.120 (10.0.10.120) 56(84) bytes of data. ^C --- 10.0.10.120 ping statistics --- 3 packets transmitted, 0 received, 100% packet loss, time 2047ms [root@bogon ~]# ip netns exec ns1 ping 10.0.10.120 PING 10.0.10.120 (10.0.10.120) 56(84) bytes of data. ^C --- 10.0.10.120 ping statistics --- 3 packets transmitted, 0 received, 100% packet loss, time 2097ms [root@bogon ~]#
添加单播
[root@kunpeng82 devuser]# vtep-ctl add-ucast-remote ls0 d2:40:3e:24:38:f6 192.168.16.81 [root@kunpeng82 devuser]# vtep-ctl list-remote-macs ls0 ucast-mac-remote d2:40:3e:24:38:f6 -> vxlan_over_ipv4/192.168.16.81 mcast-mac-remote [root@kunpeng82 devuser]#
[root@bogon ~]# ip netns exec ns1 ping 10.0.10.120 PING 10.0.10.120 (10.0.10.120) 56(84) bytes of data. 64 bytes from 10.0.10.120: icmp_seq=1 ttl=64 time=0.838 ms 64 bytes from 10.0.10.120: icmp_seq=2 ttl=64 time=0.175 ms 64 bytes from 10.0.10.120: icmp_seq=3 ttl=64 time=0.135 ms 64 bytes from 10.0.10.120: icmp_seq=4 ttl=64 time=0.130 ms 64 bytes from 10.0.10.120: icmp_seq=5 ttl=64 time=0.121 ms 64 bytes from 10.0.10.120: icmp_seq=6 ttl=64 time=0.120 ms ^C --- 10.0.10.120 ping statistics --- 6 packets transmitted, 6 received, 0% packet loss, time 5207ms rtt min/avg/max/mdev = 0.120/0.253/0.838/0.262 ms [root@bogon ~]#
[root@kunpeng82 devuser]# ovs-vsctl show ec7f002f-352c-476c-8348-5f2c86c8194f Bridge "br0_vtep_ls1" Port "vx3" Interface "vx3" type: vxlan options: {key="33", remote_ip="192.168.16.81"} Port "br0_vtep_ls1" Interface "br0_vtep_ls1" type: internal Port "0000-tortap1-l" Interface "0000-tortap1-l" type: patch options: {peer="0000-tortap1-p"} Bridge vtep_bfd Port "bfd192.168.16.81" Interface "bfd192.168.16.81" type: vxlan options: {remote_ip="192.168.16.81"} Port vtep_bfd Interface vtep_bfd type: internal Bridge "br0" Port "br0" Interface "br0" type: internal Port "tortap1" Interface "tortap1" Port "0000-tortap1-p" Interface "0000-tortap1-p" type: patch options: {peer="0000-tortap1-l"} ovs_version: "2.12.0" [root@kunpeng82 devuser]#
[root@kunpeng82 devuser]# ovs-ofctl dump-flows br0 cookie=0x0, duration=63296.064s, table=0, n_packets=74, n_bytes=4256, in_port=tortap1 actions=output:"0000-tortap1-p" cookie=0x0, duration=63296.057s, table=0, n_packets=29, n_bytes=2454, in_port="0000-tortap1-p" actions=output:tortap1
[root@kunpeng82 devuser]# ovs-ofctl dump-flows vtep_bfd cookie=0x0, duration=67326.434s, table=0, n_packets=0, n_bytes=0, priority=0 actions=NORMAL
[root@kunpeng82 devuser]# ovs-ofctl dump-flows br0_vtep_ls1 cookie=0x0, duration=63310.378s, table=0, n_packets=74, n_bytes=4256, in_port="0000-tortap1-l" actions=learn(table=1,idle_timeout=15,priority=1000,cookie=0x5000,NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],output:NXM_OF_IN_PORT[]),resubmit(,1) cookie=0x0, duration=313.533s, table=0, n_packets=9, n_bytes=742, priority=1000,in_port=vx3 actions=resubmit(,1) cookie=0x0, duration=63310.458s, table=0, n_packets=0, n_bytes=0, priority=0 actions=drop cookie=0x0, duration=313.526s, table=1, n_packets=8, n_bytes=672, priority=1000,dl_dst=d2:40:3e:24:38:f6 actions=output:vx3 cookie=0x0, duration=11928.008s, table=1, n_packets=0, n_bytes=0, priority=1,in_port=2 actions=output:"0000-tortap1-l" cookie=0x0, duration=11901.341s, table=1, n_packets=4, n_bytes=252, priority=1,in_port=3 actions=output:"0000-tortap1-l" cookie=0x0, duration=450.907s, table=1, n_packets=2, n_bytes=112, priority=0 actions=output:"0000-tortap1-l" [root@kunpeng82 devuser]#
vtep-ctl add-ls ls0 vtep-ctl bind-ls br0 tortap1 0 ls0 vtep-ctl set Logical_Switch ls0 tunnel_key=33 vtep-ctl add-mcast-remote ls0 unknown-dst 192.168.16.81