Zookeeper实现分布式锁，下面接着记录下代码实现，代码部分参考某网络课程，其他部分参考文末博文。
实现思路
以下是大方向的实现思路，相比上篇区分读写请求，这里代码实现是不区分读写的，直接只比较/shared节点下临时有序节点的序号，最小的将获取到锁，并且设置的Watcher监听也只会监听前一个序号对应的节点。
代码实现
自定义一个类，实现Watcher接口，重写里面的回调函数process(WatchedEvent event)，根据回调参数event返回的不同结果，实行不同的逻辑处理。
ZKDistributedLock类
package com.boe.lock;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.Logger;
import org.apache.zookeeper.*;
import org.apache.zookeeper.data.Stat;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
/**
* 自定义分布式锁
*/
public class ZKDistributedLock implements Watcher {
private int threadId;
//形如[第i个线程]
private String THREAD_FLAG;
//连接zookeeper服务器用
private ZooKeeper zk = null;
//分布式锁相关父节点路径
private static final String GROUP_PATH = "/shared";
//分布式锁相关的子节点路径
private static final String SUB_PATH = "/shared/node";
//当前临时顺序子节点
private String currentEphemeralNode;
//比当前临时顺序子节点序号小的前面那个子节点
private String preEphemeralNode;
//打印日志相关
private static final Logger logger = Logger.getLogger(ZKDistributedLock.class);
//使用闭锁，必须计数为0，await方法才解除阻塞继续向下执行
private CountDownLatch connectCountDownLatch = new CountDownLatch(1);
//使用闭锁，应对10个线程都抢到锁并处理完逻辑后，打印结果
public static final CountDownLatch threadCountDownLatch = new CountDownLatch(5);
//记录process方法使用次数
private int count=0;
/**
* 构造方法
*/
public ZKDistributedLock(int threadId) {
this.threadId = threadId;
THREAD_FLAG = "[第" + threadId + "个线程]";
}
/**
* 回调函数，返回连接结果
*
* @param event
*/
@Override
public void process(WatchedEvent event) {
if (event == null) {
return;
}
//一种跟连接相关
Event.KeeperState connectState = event.getState();
//一种跟事件类型相关，如获取锁的线程删除临时节点，释放锁
Event.EventType eventType = event.getType();
//必须同时满足两个条件，闭锁才能减1
if (connectState == Event.KeeperState.SyncConnected && eventType == Event.EventType.None) {
//连接zookeeper成功
logger.info(THREAD_FLAG + "连接zookeeper服务器成功");
//闭锁减一
connectCountDownLatch.countDown();
} else if (event.getType() == Event.EventType.NodeDeleted && event.getPath().equals(preEphemeralNode)) {
logger.info(THREAD_FLAG+"当前节点"+currentEphemeralNode+"前面节点"+preEphemeralNode+"被删除");
try {
if (isMinPath()) {
getDistributedLock();
}
} catch (KeeperException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
} else if (Event.KeeperState.Disconnected == connectState) {
logger.info(THREAD_FLAG + "与zookeeper服务器断开连接");
} else if (Event.KeeperState.AuthFailed == connectState) {
logger.info(THREAD_FLAG + "权限检查失败");
} else if (Event.KeeperState.Expired == connectState) {
logger.info(THREAD_FLAG + "与zookeeper服务器会话失效");
}
System.out.println(THREAD_FLAG+"执行"+(++count)+"次process方法");
}
/**
* 创建跟zookeeper服务器的连接
*
* @param connectAddress zookeeper服务器地址
* @param sessionTimeout session超时时间
*/
public void createConnection(String connectAddress, int sessionTimeout) throws IOException, InterruptedException {
zk = new ZooKeeper(connectAddress, sessionTimeout, this);
System.out.println(THREAD_FLAG+"开始连接zookeeper...");
connectCountDownLatch.await();
//回调函数闭锁归零，才打印
//System.out.println("连接zookeeper成功！");
}
/**
* 创建组路径
*/
public void createGroupPath() throws KeeperException, InterruptedException {
//zookeeper中不存在就创建
if (zk.exists(GROUP_PATH, true) == null) {
String createdPath = zk.create(GROUP_PATH, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
logger.warn(THREAD_FLAG + "创建" + createdPath + "成功");
}
}
/**
* 尝试获取锁
*/
public void grabLock() throws KeeperException, InterruptedException {
//先创建临时节点
currentEphemeralNode = zk.create(SUB_PATH, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
logger.info(THREAD_FLAG + "创建" + currentEphemeralNode+"临时顺序节点");
//检查是否可以获取锁
if (isMinPath()) {
//如果是最小序号的节点，获取锁
getDistributedLock();
}
}
/**
* 判断当前临时节点是否是最小序号的，如果不是最小节点，获取前面节点，并且设置监听
* @return
*/
private boolean isMinPath() throws KeeperException, InterruptedException {
//获取所有节点，并且排序
List<String> nodes = zk.getChildren(GROUP_PATH, false);
Collections.sort(nodes);
//判断当前创建的临时顺序节点是否是序号为0的，是0就是最小，可以获取锁
String node = currentEphemeralNode.substring(GROUP_PATH.length() + 1);//形如node0000?
int index = nodes.indexOf(node);
switch (index) {
case -1:
logger.info(THREAD_FLAG + "本节点不存在了" + node);
return false;
case 0:
logger.info(THREAD_FLAG +node+"就是最小的子节点");
return true;
default:
try {
//找到比自己序号小1的临时节点
preEphemeralNode = GROUP_PATH + "/" + nodes.get(index - 1);
logger.info(THREAD_FLAG + currentEphemeralNode + "前面的节点是" + preEphemeralNode);
//通过查询节点数据来设置监听
zk.getData(preEphemeralNode, true, new Stat());
return false;
} catch (KeeperException e) {
if (zk.exists(preEphemeralNode, false) == null) {
logger.info(THREAD_FLAG + "排在" + currentEphemeralNode + "前面的子节点" + preEphemeralNode + "不存在");
//出现异常，再次调用
return isMinPath();
} else {
throw e;
}
}
}
}
/**
* 获取锁成功
*/
private void getDistributedLock() throws KeeperException, InterruptedException {
if (zk.exists(currentEphemeralNode, false) == null) {
logger.info(THREAD_FLAG + "节点不存在");
return;
}
logger.info(THREAD_FLAG + "获取锁成功");
Thread.sleep(2000);
System.out.println(THREAD_FLAG + "干完活了~~~");
//删除临时节点，不考虑版本
zk.delete(currentEphemeralNode, -1);
logger.info(THREAD_FLAG+"节点"+currentEphemeralNode+"已删除");
//释放与zookeeper服务器的连接
if (zk != null) {
zk.close();
}
logger.info(THREAD_FLAG + "释放了与zookeeper的连接");
//闭锁减1
threadCountDownLatch.countDown();
}
}
代码实现具体逻辑图，如下图所示，需要区分两种情况，一种是当前线程创建的节点就是序号最小的，一种是序号非最小的。
ZKDistributedLock类实现了Watcher接口，需重写proces方法，每个线程会执行代码里process方法2次，一次为连接zookeeper服务器成功会执行一次，一次为前一个设置Watcher监听的节点被删除后，监听到后会再次执行一次process方法，并且在process方法里完成判断是否序号最小节点(isMinPath方法)和抢占锁(getDistributedLock方法)业务逻辑。
另外，刚开始创建序号最小节点的线程，会先创建/shared父节点，因此它也会执行两次process方法，虽然也是两次但是和其他序号的子节点是走不同的路线。它会执行在grabLock方法里完成判断是否序号最小节点(isMinPath方法)和抢占锁(getDistributedLock方法)业务逻辑。
再写一个测试类，在main方法里创建5个线程，模拟5个client连接zookeeper服务器创建临时有序节点，完成抢占锁的逻辑。
TestZKDistributedLock类
package com.boe.lock;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.Logger;
import java.util.concurrent.CountDownLatch;
/**
* 测试zookeeper分布式锁
*/
public class TestZKDistributedLock {
private int threadId;
//连接zookeeper的地址
private static final String CONNECT_ADDRESS="node01:2181,node02:2181,node03:2181";
private static final int SESSION_TIMEOUT=30000;
//打印日志相关
private static final Logger logger=Logger.getLogger(TestZKDistributedLock.class);
//主方法
public static void main(String[] args) {
BasicConfigurator.configure();
//启动5个线程，模拟5个client连接zookeeper服务器，创建临时有序节点并抢占锁
for (int i = 0; i < 5; i++) {
//线程数从1开始
int threadId=i+1;
//启动线程
new Thread(new Runnable() {
@Override
public void run() {
try {
//创建跟zookeeper服务器的连接
ZKDistributedLock lock=new ZKDistributedLock(threadId);
lock.createConnection(CONNECT_ADDRESS,SESSION_TIMEOUT);
System.out.println("[第"+threadId+"个线程]+"连接成功，准备开始创建临时有序节点和抢占锁");
//首先确认是否创建父节点/Shared,只能有一个线程创建成功,zookeeper只会让一个线程创建成功
//哪个线程先执行到这里，哪个线程就创建父节点
synchronized (ZKDistributedLock.threadCountDownLatch){
lock.createGroupPath();
}
//尝试获取分布式锁
lock.grabLock();
} catch (Exception e) {
logger.error("[第"+threadId+"个线程]抛出异常");
e.printStackTrace();
}
}
}).start();
}
//循环结束，提示所有线程执行完成
try {
ZKDistributedLock.threadCountDownLatch.await();
logger.info("所有线程创建临时有序节点完成，并完成抢占锁后执行业务逻辑");
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
测试结果
启动zookeeper服务器，IDEA中本地运行，通过查看控制台日志记录，以及连接zookeeper服务器查看节点下子节点情况，发现正常的执行了创建临时有序节点、判断是否是最小节点、在前一个节点设置Watcher监听、抢占锁、释放锁的业务。
（1）IDEA中控制台中结果
# 模拟5个client连接zookeeper服务器
[第5个线程]开始连接zookeeper...
[第1个线程]开始连接zookeeper...
[第2个线程]开始连接zookeeper...
[第4个线程]开始连接zookeeper...
[第3个线程]开始连接zookeeper...
# 5个线程均连接成功，都会执行一次process方法
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(74) -[第5个线程]连接zookeeper服务器成功
264 [Thread-4-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第5个线程]连接zookeeper服务器成功
[第5个线程]执行1次process方法
[第5个线程]连接成功，准备开始创建临时有序节点和抢占锁
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(74) -[第3个线程]连接zookeeper服务器成功
264 [Thread-2-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第3个线程]连接zookeeper服务器成功
[第3个线程]执行1次process方法
[第3个线程]连接成功，准备开始创建临时有序节点和抢占锁
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(74) -[第4个线程]连接zookeeper服务器成功
264 [Thread-3-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第4个线程]连接zookeeper服务器成功
[第4个线程]执行1次process方法
[第4个线程]连接成功，准备开始创建临时有序节点和抢占锁
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(74) -[第2个线程]连接zookeeper服务器成功
264 [Thread-1-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第2个线程]连接zookeeper服务器成功
[第2个线程]执行1次process方法
[第2个线程]连接成功，准备开始创建临时有序节点和抢占锁
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(74) -[第1个线程]连接zookeeper服务器成功
264 [Thread-0-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第1个线程]连接zookeeper服务器成功
[第1个线程]执行1次process方法
[第1个线程]连接成功，准备开始创建临时有序节点和抢占锁
# 创建/shared父节点的线程，会再次执行一次process方法，它也将创建最小序号的临时有序节点
[第5个线程]执行2次process方法
323 [Thread-4-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940019, packet:: clientPath:null serverPath:null finished:false header:: 2,1 replyHeader:: 2,257698038051,0 request:: '/shared,,v{s{31,s{'world,'anyone}}},0 response:: '/shared
2020-08-30 14:50:02 [ WARN] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(122) -[第5个线程]创建/shared成功
323 [Thread-4] WARN com.kaikeba.lock.ZKDistributedLock - [第5个线程]创建/shared成功
340 [Thread-4-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940019, packet:: clientPath:null serverPath:null finished:false header:: 3,1 replyHeader:: 3,257698038052,0 request:: '/shared/node,,v{s{31,s{'world,'anyone}}},3 response:: '/shared/node0000000000
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(133) -[第5个线程]创建/shared/node0000000000临时顺序节点
340 [Thread-4] INFO com.kaikeba.lock.ZKDistributedLock - [第5个线程]创建/shared/node0000000000临时顺序节点
344 [Thread-0-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940016, packet:: clientPath:null serverPath:null finished:false header:: 1,3 replyHeader:: 1,257698038052,0 request:: '/shared,T response:: s{257698038051,257698038051,1598770202917,1598770202917,0,1,0,0,0,1,257698038052}
364 [Thread-4-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940019, packet:: clientPath:null serverPath:null finished:false header:: 4,8 replyHeader:: 4,257698038052,0 request:: '/shared,F response:: v{'node0000000000}
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(158) -[第5个线程]node0000000000就是最小的子节点
365 [Thread-4] INFO com.kaikeba.lock.ZKDistributedLock - [第5个线程]node0000000000就是最小的子节点
369 [Thread-0-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940016, packet:: clientPath:null serverPath:null finished:false header:: 2,1 replyHeader:: 2,257698038053,0 request:: '/shared/node,,v{s{31,s{'world,'anyone}}},3 response:: '/shared/node0000000001
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(133) -[第1个线程]创建/shared/node0000000001临时顺序节点
369 [Thread-0] INFO com.kaikeba.lock.ZKDistributedLock - [第1个线程]创建/shared/node0000000001临时顺序节点
369 [Thread-4-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940019, packet:: clientPath:null serverPath:null finished:false header:: 5,3 replyHeader:: 5,257698038053,0 request:: '/shared/node0000000000,F response:: s{257698038052,257698038052,1598770202930,1598770202930,0,0,0,176833595994013721,0,0,257698038052}
369 [Thread-1-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940018, packet:: clientPath:null serverPath:null finished:false header:: 1,3 replyHeader:: 1,257698038053,0 request:: '/shared,T response:: s{257698038051,257698038051,1598770202917,1598770202917,0,2,0,0,0,2,257698038053}
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(188) -[第5个线程]获取锁成功
370 [Thread-4] INFO com.kaikeba.lock.ZKDistributedLock - [第5个线程]获取锁成功
376 [Thread-3-SendThread(node03:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x3743d37f63e001a, packet:: clientPath:null serverPath:null finished:false header:: 1,3 replyHeader:: 1,257698038053,0 request:: '/shared,T response:: s{257698038051,257698038051,1598770202917,1598770202917,0,2,0,0,0,2,257698038053}
380 [Thread-0-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940016, packet:: clientPath:null serverPath:null finished:false header:: 3,8 replyHeader:: 3,257698038053,0 request:: '/shared,F response:: v{'node0000000000,'node0000000001}
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(164) -[第1个线程]/shared/node0000000001前面的节点是/shared/node0000000000
380 [Thread-0] INFO com.kaikeba.lock.ZKDistributedLock - [第1个线程]/shared/node0000000001前面的节点是/shared/node0000000000
383 [Thread-1-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940018, packet:: clientPath:null serverPath:null finished:false header:: 2,1 replyHeader:: 2,257698038054,0 request:: '/shared/node,,v{s{31,s{'world,'anyone}}},3 response:: '/shared/node0000000002
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(133) -[第2个线程]创建/shared/node0000000002临时顺序节点
383 [Thread-1] INFO com.kaikeba.lock.ZKDistributedLock - [第2个线程]创建/shared/node0000000002临时顺序节点
384 [Thread-2-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940017, packet:: clientPath:null serverPath:null finished:false header:: 1,3 replyHeader:: 1,257698038054,0 request:: '/shared,T response:: s{257698038051,257698038051,1598770202917,1598770202917,0,3,0,0,0,3,257698038054}
391 [Thread-1-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940018, packet:: clientPath:null serverPath:null finished:false header:: 3,8 replyHeader:: 3,257698038055,0 request:: '/shared,F response:: v{'node0000000002,'node0000000003,'node0000000000,'node0000000001}
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(164) -[第2个线程]/shared/node0000000002前面的节点是/shared/node0000000001
391 [Thread-1] INFO com.kaikeba.lock.ZKDistributedLock - [第2个线程]/shared/node0000000002前面的节点是/shared/node0000000001
392 [Thread-0-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940016, packet:: clientPath:null serverPath:null finished:false header:: 4,4 replyHeader:: 4,257698038055,0 request:: '/shared/node0000000000,T response:: ,s{257698038052,257698038052,1598770202930,1598770202930,0,0,0,176833595994013721,0,0,257698038052}
397 [Thread-3-SendThread(node03:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x3743d37f63e001a, packet:: clientPath:null serverPath:null finished:false header:: 2,1 replyHeader:: 2,257698038055,0 request:: '/shared/node,,v{s{31,s{'world,'anyone}}},3 response:: '/shared/node0000000003
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(133) -[第4个线程]创建/shared/node0000000003临时顺序节点
397 [Thread-3] INFO com.kaikeba.lock.ZKDistributedLock - [第4个线程]创建/shared/node0000000003临时顺序节点
400 [Thread-1-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940018, packet:: clientPath:null serverPath:null finished:false header:: 4,4 replyHeader:: 4,257698038056,0 request:: '/shared/node0000000001,T response:: ,s{257698038053,257698038053,1598770202953,1598770202953,0,0,0,176833595994013718,0,0,257698038053}
400 [Thread-2-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940017, packet:: clientPath:null serverPath:null finished:false header:: 2,1 replyHeader:: 2,257698038056,0 request:: '/shared/node,,v{s{31,s{'world,'anyone}}},3 response:: '/shared/node0000000004
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(133) -[第3个线程]创建/shared/node0000000004临时顺序节点
400 [Thread-2] INFO com.kaikeba.lock.ZKDistributedLock - [第3个线程]创建/shared/node0000000004临时顺序节点
404 [Thread-3-SendThread(node03:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x3743d37f63e001a, packet:: clientPath:null serverPath:null finished:false header:: 3,8 replyHeader:: 3,257698038055,0 request:: '/shared,F response:: v{'node0000000002,'node0000000003,'node0000000000,'node0000000001}
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(164) -[第4个线程]/shared/node0000000003前面的节点是/shared/node0000000002
404 [Thread-3] INFO com.kaikeba.lock.ZKDistributedLock - [第4个线程]/shared/node0000000003前面的节点是/shared/node0000000002
408 [Thread-3-SendThread(node03:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x3743d37f63e001a, packet:: clientPath:null serverPath:null finished:false header:: 4,4 replyHeader:: 4,257698038056,0 request:: '/shared/node0000000002,T response:: ,s{257698038054,257698038054,1598770202976,1598770202976,0,0,0,176833595994013720,0,0,257698038054}
408 [Thread-2-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940017, packet:: clientPath:null serverPath:null finished:false header:: 3,8 replyHeader:: 3,257698038056,0 request:: '/shared,F response:: v{'node0000000002,'node0000000003,'node0000000004,'node0000000000,'node0000000001}
2020-08-30 14:50:02 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(164) -[第3个线程]/shared/node0000000004前面的节点是/shared/node0000000003
408 [Thread-2] INFO com.kaikeba.lock.ZKDistributedLock - [第3个线程]/shared/node0000000004前面的节点是/shared/node0000000003
411 [Thread-2-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940017, packet:: clientPath:null serverPath:null finished:false header:: 4,4 replyHeader:: 4,257698038056,0 request:: '/shared/node0000000003,T response:: ,s{257698038055,257698038055,1598770202981,1598770202981,0,0,0,248891189975973914,0,0,257698038055}
# 序号最小的节点对应线程执行完逻辑，将释放分布式锁
[第5个线程]干完活了~~~
2377 [Thread-0-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Got notification sessionid:0x2743d37f9940016
2377 [Thread-0-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/shared/node0000000000 for sessionid 0x2743d37f9940016
2020-08-30 14:50:04 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(78) -[第1个线程]当前节点/shared/node0000000001前面节点/shared/node0000000000被删除
2378 [Thread-0-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第1个线程]当前节点/shared/node0000000001前面节点/shared/node0000000000被删除
2378 [Thread-4-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940019, packet:: clientPath:null serverPath:null finished:false header:: 6,2 replyHeader:: 6,257698038057,0 request:: '/shared/node0000000000,-1 response:: null
2020-08-30 14:50:04 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(193) -[第5个线程]节点/shared/node0000000000已删除
2379 [Thread-4] INFO com.kaikeba.lock.ZKDistributedLock - [第5个线程]节点/shared/node0000000000已删除
2379 [Thread-4] DEBUG org.apache.zookeeper.ZooKeeper - Closing session: 0x2743d37f9940019
2379 [Thread-4] DEBUG org.apache.zookeeper.ClientCnxn - Closing client for session: 0x2743d37f9940019
2385 [Thread-0-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940016, packet:: clientPath:null serverPath:null finished:false header:: 5,8 replyHeader:: 5,257698038057,0 request:: '/shared,F response:: v{'node0000000002,'node0000000003,'node0000000004,'node0000000001}
2020-08-30 14:50:04 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(158) -[第1个线程]node0000000001就是最小的子节点
2385 [Thread-0-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第1个线程]node0000000001就是最小的子节点
2387 [Thread-4-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940019, packet:: clientPath:null serverPath:null finished:false header:: 7,-11 replyHeader:: 7,257698038058,0 request:: null response:: null
2387 [Thread-4] DEBUG org.apache.zookeeper.ClientCnxn - Disconnecting client for session: 0x2743d37f9940019
2020-08-30 14:50:04 [ INFO] - org.apache.zookeeper.ZooKeeper -ZooKeeper.java(693) -Session: 0x2743d37f9940019 closed
2387 [Thread-4] INFO org.apache.zookeeper.ZooKeeper - Session: 0x2743d37f9940019 closed
2020-08-30 14:50:04 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(198) -[第5个线程]释放了与zookeeper的连接
2388 [Thread-4] INFO com.kaikeba.lock.ZKDistributedLock - [第5个线程]释放了与zookeeper的连接
2020-08-30 14:50:04 [ INFO] - org.apache.zookeeper.ClientCnxn -ClientCnxn.java(522) -EventThread shut down for session: 0x2743d37f9940019
2390 [Thread-4-EventThread] INFO org.apache.zookeeper.ClientCnxn - EventThread shut down for session: 0x2743d37f9940019
2393 [Thread-0-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940016, packet:: clientPath:null serverPath:null finished:false header:: 6,3 replyHeader:: 6,257698038058,0 request:: '/shared/node0000000001,F response:: s{257698038053,257698038053,1598770202953,1598770202953,0,0,0,176833595994013718,0,0,257698038053}
2020-08-30 14:50:04 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(188) -[第1个线程]获取锁成功
2394 [Thread-0-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第1个线程]获取锁成功
[第1个线程]干完活了~~~
4402 [Thread-1-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Got notification sessionid:0x2743d37f9940018
4403 [Thread-1-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/shared/node0000000001 for sessionid 0x2743d37f9940018
2020-08-30 14:50:06 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(78) -[第2个线程]当前节点/shared/node0000000002前面节点/shared/node0000000001被删除
4403 [Thread-1-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第2个线程]当前节点/shared/node0000000002前面节点/shared/node0000000001被删除
4403 [Thread-0-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940016, packet:: clientPath:null serverPath:null finished:false header:: 7,2 replyHeader:: 7,257698038059,0 request:: '/shared/node0000000001,-1 response:: null
2020-08-30 14:50:06 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(193) -[第1个线程]节点/shared/node0000000001已删除
4403 [Thread-0-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第1个线程]节点/shared/node0000000001已删除
4404 [Thread-0-EventThread] DEBUG org.apache.zookeeper.ZooKeeper - Closing session: 0x2743d37f9940016
4404 [Thread-0-EventThread] DEBUG org.apache.zookeeper.ClientCnxn - Closing client for session: 0x2743d37f9940016
4405 [Thread-1-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940018, packet:: clientPath:null serverPath:null finished:false header:: 5,8 replyHeader:: 5,257698038059,0 request:: '/shared,F response:: v{'node0000000002,'node0000000003,'node0000000004}
2020-08-30 14:50:06 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(158) -[第2个线程]node0000000002就是最小的子节点
4406 [Thread-1-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第2个线程]node0000000002就是最小的子节点
4408 [Thread-0-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940016, packet:: clientPath:null serverPath:null finished:false header:: 8,-11 replyHeader:: 8,257698038060,0 request:: null response:: null
4408 [Thread-0-EventThread] DEBUG org.apache.zookeeper.ClientCnxn - Disconnecting client for session: 0x2743d37f9940016
2020-08-30 14:50:06 [ INFO] - org.apache.zookeeper.ZooKeeper -ZooKeeper.java(693) -Session: 0x2743d37f9940016 closed
4408 [Thread-0-EventThread] INFO org.apache.zookeeper.ZooKeeper - Session: 0x2743d37f9940016 closed
2020-08-30 14:50:06 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(198) -[第1个线程]释放了与zookeeper的连接
4409 [Thread-0-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第1个线程]释放了与zookeeper的连接
# 刚开始不是最小序号的节点，需要在再次执行process方法里抢占锁
[第1个线程]执行2次process方法
2020-08-30 14:50:06 [ INFO] - org.apache.zookeeper.ClientCnxn -ClientCnxn.java(522) -EventThread shut down for session: 0x2743d37f9940016
4409 [Thread-0-EventThread] INFO org.apache.zookeeper.ClientCnxn - EventThread shut down for session: 0x2743d37f9940016
4410 [Thread-1-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940018, packet:: clientPath:null serverPath:null finished:false header:: 6,3 replyHeader:: 6,257698038060,0 request:: '/shared/node0000000002,F response:: s{257698038054,257698038054,1598770202976,1598770202976,0,0,0,176833595994013720,0,0,257698038054}
2020-08-30 14:50:06 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(188) -[第2个线程]获取锁成功
4410 [Thread-1-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第2个线程]获取锁成功
[第2个线程]干完活了~~~
6414 [Thread-3-SendThread(node03:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Got notification sessionid:0x3743d37f63e001a
6414 [Thread-3-SendThread(node03:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/shared/node0000000002 for sessionid 0x3743d37f63e001a
6415 [Thread-1-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940018, packet:: clientPath:null serverPath:null finished:false header:: 7,2 replyHeader:: 7,257698038061,0 request:: '/shared/node0000000002,-1 response:: null
2020-08-30 14:50:08 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(193) -[第2个线程]节点/shared/node0000000002已删除
6415 [Thread-1-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第2个线程]节点/shared/node0000000002已删除
6415 [Thread-1-EventThread] DEBUG org.apache.zookeeper.ZooKeeper - Closing session: 0x2743d37f9940018
6415 [Thread-1-EventThread] DEBUG org.apache.zookeeper.ClientCnxn - Closing client for session: 0x2743d37f9940018
2020-08-30 14:50:08 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(78) -[第4个线程]当前节点/shared/node0000000003前面节点/shared/node0000000002被删除
6418 [Thread-3-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第4个线程]当前节点/shared/node0000000003前面节点/shared/node0000000002被删除
6420 [Thread-1-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940018, packet:: clientPath:null serverPath:null finished:false header:: 8,-11 replyHeader:: 8,257698038062,0 request:: null response:: null
6420 [Thread-1-EventThread] DEBUG org.apache.zookeeper.ClientCnxn - Disconnecting client for session: 0x2743d37f9940018
2020-08-30 14:50:08 [ INFO] - org.apache.zookeeper.ZooKeeper -ZooKeeper.java(693) -Session: 0x2743d37f9940018 closed
6420 [Thread-1-EventThread] INFO org.apache.zookeeper.ZooKeeper - Session: 0x2743d37f9940018 closed
2020-08-30 14:50:08 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(198) -[第2个线程]释放了与zookeeper的连接
6420 [Thread-1-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第2个线程]释放了与zookeeper的连接
[第2个线程]执行2次process方法
2020-08-30 14:50:08 [ INFO] - org.apache.zookeeper.ClientCnxn -ClientCnxn.java(522) -EventThread shut down for session: 0x2743d37f9940018
6421 [Thread-1-EventThread] INFO org.apache.zookeeper.ClientCnxn - EventThread shut down for session: 0x2743d37f9940018
6422 [Thread-3-SendThread(node03:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x3743d37f63e001a, packet:: clientPath:null serverPath:null finished:false header:: 5,8 replyHeader:: 5,257698038062,0 request:: '/shared,F response:: v{'node0000000003,'node0000000004}
2020-08-30 14:50:08 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(158) -[第4个线程]node0000000003就是最小的子节点
6422 [Thread-3-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第4个线程]node0000000003就是最小的子节点
6425 [Thread-3-SendThread(node03:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x3743d37f63e001a, packet:: clientPath:null serverPath:null finished:false header:: 6,3 replyHeader:: 6,257698038062,0 request:: '/shared/node0000000003,F response:: s{257698038055,257698038055,1598770202981,1598770202981,0,0,0,248891189975973914,0,0,257698038055}
2020-08-30 14:50:08 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(188) -[第4个线程]获取锁成功
6425 [Thread-3-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第4个线程]获取锁成功
[第4个线程]干完活了~~~
8433 [Thread-2-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Got notification sessionid:0x2743d37f9940017
8434 [Thread-2-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/shared/node0000000003 for sessionid 0x2743d37f9940017
2020-08-30 14:50:10 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(78) -[第3个线程]当前节点/shared/node0000000004前面节点/shared/node0000000003被删除
8434 [Thread-2-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第3个线程]当前节点/shared/node0000000004前面节点/shared/node0000000003被删除
8434 [Thread-3-SendThread(node03:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x3743d37f63e001a, packet:: clientPath:null serverPath:null finished:false header:: 7,2 replyHeader:: 7,257698038063,0 request:: '/shared/node0000000003,-1 response:: null
2020-08-30 14:50:10 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(193) -[第4个线程]节点/shared/node0000000003已删除
8434 [Thread-3-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第4个线程]节点/shared/node0000000003已删除
8434 [Thread-3-EventThread] DEBUG org.apache.zookeeper.ZooKeeper - Closing session: 0x3743d37f63e001a
8434 [Thread-3-EventThread] DEBUG org.apache.zookeeper.ClientCnxn - Closing client for session: 0x3743d37f63e001a
8436 [Thread-2-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940017, packet:: clientPath:null serverPath:null finished:false header:: 5,8 replyHeader:: 5,257698038063,0 request:: '/shared,F response:: v{'node0000000004}
2020-08-30 14:50:10 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(158) -[第3个线程]node0000000004就是最小的子节点
8436 [Thread-2-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第3个线程]node0000000004就是最小的子节点
8439 [Thread-3-SendThread(node03:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x3743d37f63e001a, packet:: clientPath:null serverPath:null finished:false header:: 8,-11 replyHeader:: 8,257698038064,0 request:: null response:: null
8439 [Thread-3-EventThread] DEBUG org.apache.zookeeper.ClientCnxn - Disconnecting client for session: 0x3743d37f63e001a
8439 [Thread-2-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940017, packet:: clientPath:null serverPath:null finished:false header:: 6,3 replyHeader:: 6,257698038064,0 request:: '/shared/node0000000004,F response:: s{257698038056,257698038056,1598770202989,1598770202989,0,0,0,176833595994013719,0,0,257698038056}
2020-08-30 14:50:10 [ INFO] - org.apache.zookeeper.ZooKeeper -ZooKeeper.java(693) -Session: 0x3743d37f63e001a closed
8439 [Thread-3-EventThread] INFO org.apache.zookeeper.ZooKeeper - Session: 0x3743d37f63e001a closed
2020-08-30 14:50:10 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(198) -[第4个线程]释放了与zookeeper的连接
8440 [Thread-3-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第4个线程]释放了与zookeeper的连接
[第4个线程]执行2次process方法
2020-08-30 14:50:10 [ INFO] - org.apache.zookeeper.ClientCnxn -ClientCnxn.java(522) -EventThread shut down for session: 0x3743d37f63e001a
8440 [Thread-3-EventThread] INFO org.apache.zookeeper.ClientCnxn - EventThread shut down for session: 0x3743d37f63e001a
2020-08-30 14:50:10 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(188) -[第3个线程]获取锁成功
8440 [Thread-2-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第3个线程]获取锁成功
[第3个线程]干完活了~~~
10452 [Thread-2-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940017, packet:: clientPath:null serverPath:null finished:false header:: 7,2 replyHeader:: 7,257698038065,0 request:: '/shared/node0000000004,-1 response:: null
2020-08-30 14:50:12 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(193) -[第3个线程]节点/shared/node0000000004已删除
10453 [Thread-2-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第3个线程]节点/shared/node0000000004已删除
10454 [Thread-2-EventThread] DEBUG org.apache.zookeeper.ZooKeeper - Closing session: 0x2743d37f9940017
10454 [Thread-2-EventThread] DEBUG org.apache.zookeeper.ClientCnxn - Closing client for session: 0x2743d37f9940017
10468 [Thread-2-SendThread(node02:2181)] DEBUG org.apache.zookeeper.ClientCnxn - Reading reply sessionid:0x2743d37f9940017, packet:: clientPath:null serverPath:null finished:false header:: 8,-11 replyHeader:: 8,257698038066,0 request:: null response:: null
10468 [Thread-2-EventThread] DEBUG org.apache.zookeeper.ClientCnxn - Disconnecting client for session: 0x2743d37f9940017
2020-08-30 14:50:13 [ INFO] - org.apache.zookeeper.ZooKeeper -ZooKeeper.java(693) -Session: 0x2743d37f9940017 closed
10468 [Thread-2-EventThread] INFO org.apache.zookeeper.ZooKeeper - Session: 0x2743d37f9940017 closed
2020-08-30 14:50:13 [ INFO] - com.kaikeba.lock.ZKDistributedLock -ZKDistributedLock.java(198) -[第3个线程]释放了与zookeeper的连接
10468 [Thread-2-EventThread] INFO com.kaikeba.lock.ZKDistributedLock - [第3个线程]释放了与zookeeper的连接
[第3个线程]执行2次process方法
2020-08-30 14:50:13 [ INFO] - org.apache.zookeeper.ClientCnxn -ClientCnxn.java(522) -EventThread shut down for session: 0x2743d37f9940017
10469 [Thread-2-EventThread] INFO org.apache.zookeeper.ClientCnxn - EventThread shut down for session: 0x2743d37f9940017
2020-08-30 14:50:13 [ INFO] - com.kaikeba.lock.TestZKDistributedLock -TestZKDistributedLock.java(55) -所有线程创建临时有序节点完成，并完成抢占锁后执行业务逻辑
10470 [main] INFO com.kaikeba.lock.TestZKDistributedLock - 所有线程创建临时有序节点完成，并完成抢占锁后执行业务逻辑
（2）连接zookeeper服务器查看子节点结果
# IDEA中本地启动5个线程后，创建了父节点/shared，且创建了临时有序子节点
[zk: node01:2181(CONNECTED) 104] ls /shared
[node0000000002, node0000000003, node0000000004, node0000000000, node0000000001]
[zk: node01:2181(CONNECTED) 105] ls /shared
[node0000000002, node0000000003, node0000000004, node0000000000, node0000000001]
[zk: node01:2181(CONNECTED) 106] ls /shared
[node0000000002, node0000000003, node0000000004, node0000000000, node0000000001]
[zk: node01:2181(CONNECTED) 107] ls /shared
[node0000000002, node0000000003, node0000000004, node0000000001]
[zk: node01:2181(CONNECTED) 108] ls /shared
[node0000000002, node0000000003, node0000000004, node0000000001]
[zk: node01:2181(CONNECTED) 109] ls /shared
[node0000000002, node0000000003, node0000000004]
[zk: node01:2181(CONNECTED) 110] ls /shared
[node0000000002, node0000000003, node0000000004]
[zk: node01:2181(CONNECTED) 111] ls /shared
[node0000000003, node0000000004]
[zk: node01:2181(CONNECTED) 112] ls /shared
[node0000000003, node0000000004]
[zk: node01:2181(CONNECTED) 113] ls /shared
[node0000000003, node0000000004]
[zk: node01:2181(CONNECTED) 114] ls /shared
[node0000000004]
[zk: node01:2181(CONNECTED) 115] ls /shared
[node0000000004]
[zk: node01:2181(CONNECTED) 116] ls /shared
[node0000000004]
[zk: node01:2181(CONNECTED) 117] ls /shared
[]
子节点删除的顺序按照序号从小到大，与前文的实现思路一致。
Watcher初识
实现类实现了Watcher接口，参考文末博文对它进行简单了解，更加详细的说明参考文末博文。
实现Watcher接口的类需要重写process方法，并且本文实现类ZKDistributedLock也是一个新的Watcher，它内部接口Event包含两个枚举类，一个是KeeperState，一个是EventType，在process方法中多次用到了这两个枚举类的属性进行判断。
KeeperState代表和zookeeper连接时状态发生变化对应的通知，具体属性意思代码注释如下。
//不再使用
@Deprecated
Unknown (-1),
//客户端与服务器器断开连接
Disconnected (0),
//不再使用
@Deprecated
NoSyncConnected (1),
//客户端与服务器正常连接时
SyncConnected (3),
//身份认证失败
AuthFailed (4),
//则当zk集群中只有少于半数的服务器正常时，会返回这个状态给客户端，此时客户端只能处理读请求，在3.3.0版本后才支持
ConnectedReadOnly (5),
//服务器采用SASL做校验时
SaslAuthenticated(6),
//会话session失效时
Expired (-112);
EventType是znode发生变化时对应的通知类型，具体属性意思代码注释如下。
//无
None (-1),
//Watcher监听的数据节点被创建时
NodeCreated (1),
//Watcher监听的数据节点被删除时
NodeDeleted (2),
//Watcher监听的数据节点内容发生变更时(无论内容数据是否变化)
NodeDataChanged (3),
//Watcher监听的数据节点的子节点列表发生变更时
NodeChildrenChanged (4);
以上，理解不一定正确，学习就是一个不断认识和纠错的过程，如果有误还请批评指正。
PS：7.28是一个值得纪念的日子。
参考博文：
（1）《从Paxos到Zookeeper-分布式一致性原理与实践》分布式锁
（2）https://www.jianshu.com/p/c68b6b241943 watcher机制
（3）https://www.cnblogs.com/leesf456/p/6286827.html watcher源码一