Docker 集群管理

docker systemd unit file

[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.com
After=network.target docker.socket
Requires=docker.socket

[Service]
#Environment=HTTP_PROXY=10.48.127.169:8080 HTTPS_PROXY=10.48.127.169:8080
ExecStart=/usr/bin/docker -d -H unix:///var/run/docker.sock -H 0.0.0.0:4243
LimitNOFILE=1048576
LimitNPROC=1048576
EnvironmentFile=/lib/systemd/system/docker_env

[Install]
WantedBy=multi-user.target

Docker 环境配置

vim /lib/systemd/system/docker_env

HTTP_PROXY=http://10.48.127.169:8080
HTTPS_PROXY=http://10.48.127.169:8080
NO_PROXY="10.58.9.82,10.58.9.84"

etcd是一个高可用的键值存储系统，主要用于共享配置和服务发现。etcd是由CoreOS开发并维护的，灵感来自于 ZooKeeper 和 Doozer，它使用Go语言编写，并通过Raft一致性算法处理日志复制以保证强一致性。Raft是一个来自Stanford的新的一致性算法，适用于分布式系统的日志复制，Raft通过选举的方式来实现一致性，在Raft中，任何一个节点都可能成为Leader。Google的容器集群管理系统Kubernetes、开源PaaS平台Cloud Foundry和CoreOS的Fleet都广泛使用了etcd。

安装 

git clone https://github.com/coreos/etcd.git
cd /usr/local/etcd
./bulid
ln -s /usr/local/etcd/bin/etcd /usr/local/bin/

官网建议使用0.4.6版 

cd /usr/local/etcd
git checkout -b v0.4.6
./build

 安装etcdctl

git clone https://github.com/coreos/etcdctl.git
cd /usr/local/etcdctl/
./build
ln -s /usr/local/etcdctl/bin/etcdctl /usr/local/bin/

配置

etcd discovery 

etcd -name discovery -peer-addr 10.58.9.85:8001 -peer-bind-addr 10.58.9.85:8001 -addr 10.58.9.85:5001 -bind-addr 10.58.9.85:5001 -data-dir /home/discovery -discovery=""

etcd services

vim /etc/etcd/etcd.conf

addr = "10.58.9.82:4001"
bind_addr = "0.0.0.0"
#change to every member's client IP and port 
ca_file = ""
cert_file = ""
cors = []
cpu_profile_file = ""
data_dir = "/home/etcd/machine"
#specify data directory
discovery = "http://10.58.9.85:5001/v2/keys/discovery"
#configure to discovery cluster's ip + cluster ID
http_read_timeout = 10.0
http_write_timeout = 10.0
key_file = ""
peers = []
peers_file = ""
max_cluster_size = 9
max_result_buffer = 1024
max_retry_attempts = 3
name = "Peer-82"
#change name
snapshot = true
verbose = false
very_verbose = false

[peer]
addr = "10.58.9.82:7001"
bind_addr = "0.0.0.0"
#change to every member's peer IP and port 
ca_file = ""
cert_file = ""
key_file = ""

[cluster]
active_size = 9
remove_delay = 1800.0
sync_interval = 5.0

启动etcd

etcd -config=/etc/etcd/etcd.conf

测试

curl -L http://127.0.0.1:4001/v2/keys/message -XPUT -d value="Hello world"
curl http://127.0.0.1:4001/v2/keys/message

使用systemd unit file启动

vim /lib/systemd/system/etcd-discovery.service

[Unit]
Description=Etcd Discovery Server
After=network.target

[Service]
Environment="ETCD_ADDR=10.58.9.85:5001"
Environment="ETCD_BIND_ADDR=0.0.0.0"
Environment="ETCD_DATA_DIR=/home/etcd/discovery"
Environment="ETCD_HTTP_READ_TIMEOUT=10.0"
Environment="ETCD_HTTP_WRITE_TIMEOUT=10.0"
Environment="ETCD_DISCOVERY="
Environment="ETCD_MAX_CLUSTER_SIZE=9"
Environment="ETCD_MAX_RESULT_BUFFER=1024"
Environment="ETCD_MAX_RETRY_ATTEMPTS=3"
Environment="ETCD_NAME=etcd-discovery"
Environment="ETCD_SNAPSHOT=true"
Environment="ETCD_VERBOSE=false"
Environment="ETCD_VERY_VERBOSE=false"
Environment="ETCD_PEER_ADDR=10.58.9.85:8001"
Environment="ETCD_PEER_BIND_ADDR=0.0.0.0"
Environment="ETCD_CLUSTER_ACTIVE_SIZE=9"
Environment="ETCD_CLUSTER_REMOVE_DELAY=1800.0"
Environment="ETCD_CLUSTER_SYNC_INTERVAL=5.0"

# etc logs to the journal directly, suppress double logging
#StandardOutput=null
#WorkingDirectory=/home/etcd/discovery
ExecStart=/usr/local/bin/etcd

[Install]
WantedBy=multi-user.target

vim /lib/systemd/system/etcd.service

[Unit]
Description=Etcd Server
After=network.target
#Requires=etcd-discovery.service

[Service]
Environment="ETCD_ADDR=10.58.9.85:4001"
Environment="ETCD_BIND_ADDR=0.0.0.0"
Environment="ETCD_DATA_DIR=/home/etcd/machine"
Environment="ETCD_HTTP_READ_TIMEOUT=10.0"
Environment="ETCD_HTTP_WRITE_TIMEOUT=10.0"
Environment="ETCD_DISCOVERY=http://10.58.9.85:5001/v2/keys/discovery"
Environment="ETCD_MAX_CLUSTER_SIZE=9"
Environment="ETCD_MAX_RESULT_BUFFER=1024"
Environment="ETCD_MAX_RETRY_ATTEMPTS=3"
Environment="ETCD_NAME=peer-85"
Environment="ETCD_SNAPSHOT=true"
Environment="ETCD_VERBOSE=false"
Environment="ETCD_VERY_VERBOSE=false"
Environment="ETCD_PEER_ADDR=10.58.9.85:7001"
Environment="ETCD_PEER_BIND_ADDR=0.0.0.0"
Environment="ETCD_CLUSTER_ACTIVE_SIZE=9"
Environment="ETCD_CLUSTER_REMOVE_DELAY=1800.0"
Environment="ETCD_CLUSTER_SYNC_INTERVAL=5.0"

# etc logs to the journal directly, suppress double logging
#StandardOutput=null
#WorkingDirectory=/home/etcd/discovery
ExecStart=/usr/local/bin/etcd

[Install]
WantedBy=multi-user.target

systemctl enable etcd-discovery.service
systemctl daemon-reload
systemctl start etcd-discovery.service
systemctl stop etcd-discovery.service

systemctl enable etcd.service
systemctl daemon-reload
systemctl start etcd.service
systemctl stop etcd.service

 etcdctl 命令

查看所有根节点

etcdctl ls --recursive

删除指定节点

etcdctl rm fleet --recursive

 安装fleet  

git clone https://github.com/coreos/fleet.git
cd fleet
./build
ln -s /usr/local/fleet/bin/fleetd /usr/local/bin/
ln -s /usr/local/fleet/bin/fleetctl /usr/local/bin/

生成dbus machine-id

http://dbus.freedesktop.org/doc/dbus-uuidgen.1.html

dbus-uuidgen > /var/lib/dbus/machine-id
ln -s /var/lib/dbus/machine-id /etc/machine-id

 systemd fleet 启动

vim /lib/systemd/system/fleet.service

[Unit]
Description=Fleet Server
After=network.target
Requires=etcd.service

[Service]
Environment="FLEET_ETCD_SERVERS=http://10.58.9.85:4001"
Environment="FLEET_ETCD_REQUEST=2.0"
Environment="FLEET_VERBOSITY=0"
#Environment="FLEET_PUBLIC_IP=10.58.9.85"
Environment="FLEET_METADATA='role=default'"
Environment="FLEET_ETCD_KEY_PREFIX=fleet"

ExecStart=/usr/local/bin/fleetd

[Install]
WantedBy=multi-user.target

systemctl enable fleet.service
systemctl start fleet.service

fleetctl 命令

fleetctl --debug --endpoint=http://10.58.9.85:4001 --etcd-key-prefix=/fleet/ list-machines
fleetctl --debug --endpoint=http://10.58.9.85:4001 --etcd-key-prefix=/fleet/ submit tickler.1.service 
fleetctl --debug --endpoint=http://10.58.9.85:4001 --etcd-key-prefix=/fleet/ list-unit-files
fleetctl --debug --endpoint=http://10.58.9.85:4001 --etcd-key-prefix=/fleet/ start tickler.1.service
fleetctl --debug --endpoint=http://10.58.9.85:4001 --etcd-key-prefix=/fleet/ status tickler.1.service
fleetctl --debug --endpoint=http://10.58.9.85:4001 --etcd-key-prefix=/fleet/ journal tickler.1.service

fleet API

vim /lib/systemd/system/fleet.socket

[Socket]
# Talk to the API over a Unix domain socket (default)
#ListenStream=/var/run/fleet.sock
# Talk to the API over an exposed port, uncomment to enable and choose a port
ListenStream=127.0.0.1:49155
Service=fleet.service

[Install]
WantedBy=sockets.target

systemctl daemon-reload
systemctl start fleet.socket
systemctl start fleet.service

 

Command	Desired State	Valid Previous States
fleetctl submit	inactive	(unknown)
fleetctl load	loaded	(unknown) or inactive
fleetctl start	launched	(unknown) or inactive or loaded
fleetctl stop	loaded	launched
fleetctl unload	inactive	launched or loaded
fleetctl destroy	(unknown)	launched or loaded or inactive