zoukankan      html  css  js  c++  java
  • rocketmq 多master集群部署

    rocketmq  并且编译下载

    wget  http://mirror.bit.edu.cn/apache/rocketmq/4.3.2/rocketmq-all-4.3.2-source-release.zip  
    unzip rocketmq-all-4.3.2-source-release.zip
    cd rocketmq-all-4.3.2/
    mvn -Prelease-all -DskipTests clean install -U
    cd distribution/target/apache-rocketmq

    拷贝 apache-rocketmq 到 /data/rocketmq目录下

    [root@rocketmq1 data]# cd /data/rocketmq-all-4.3.2/distribution/target/
    [root@rocketmq1 target]#  cp apache-rocketmq /data/rocketmq -ar 

    修改配置文件,以 vim /data/rocketmq/conf/2m-noslave/broker-a.properties 为例

    rocketmqHome=/data/rocketmq
    # 如果是一个nameserver,直接写一个地址即可
    namesrvAddr=172.16.230.21:9876;172.16.230.22:9876;172.16.230.23:9876
    brokerName=borker-a
    brokerClusterName=DefaultCluster
    brokerId=0
    brokerPermission=6
    #在发送消息时,自动创建服务器不存在的topic,默认创建的队列数
    defaultTopicQueueNums=4
    #是否允许 Broker 自动创建Topic,建议线下开启,线上关闭
    autoCreateTopicEnable=true
    clusterTopicEnable=true
    brokerTopicEnable=true
    #是否允许 Broker 自动创建订阅组,建议线下开启,线上关闭
    autoCreateSubscriptionGroup=true
    sendMessageThreadPoolNums=1
    pullMessageThreadPoolNums=20
    queryMessageThreadPoolNums=10
    adminBrokerThreadPoolNums=16
    clientManageThreadPoolNums=32
    consumerManageThreadPoolNums=32
    heartbeatThreadPoolNums=2
    endTransactionThreadPoolNums=12
    flushConsumerOffsetInterval=5000
    flushConsumerOffsetHistoryInterval=60000
    rejectTransactionMessage=false
    fetchNamesrvAddrByAddressServer=false
    sendThreadPoolQueueCapacity=10000
    pullThreadPoolQueueCapacity=100000
    queryThreadPoolQueueCapacity=20000
    clientManagerThreadPoolQueueCapacity=1000000
    consumerManagerThreadPoolQueueCapacity=1000000
    heartbeatThreadPoolQueueCapacity=50000
    endTransactionPoolQueueCapacity=100000
    filterServerNums=0
    longPollingEnable=true
    shortPollingTimeMills=1000
    notifyConsumerIdsChangedEnable=true
    highSpeedMode=false
    commercialEnable=true
    commercialTimerCount=1
    commercialTransCount=1
    commercialBigCount=1
    commercialBaseCount=1
    transferMsgByHeap=true
    maxDelayTime=40
    regionId=DefaultRegion
    registerBrokerTimeoutMills=6000
    slaveReadEnable=false
    disableConsumeIfConsumerReadSlowly=false
    consumerFallbehindThreshold=17179869184
    brokerFastFailureEnable=true
    waitTimeMillsInSendQueue=200
    waitTimeMillsInPullQueue=5000
    waitTimeMillsInHeartbeatQueue=31000
    waitTimeMillsInTransactionQueue=3000
    startAcceptSendRequestTimeStamp=0
    traceOn=true
    enableCalcFilterBitMap=false
    expectConsumerNumUseFilter=32
    maxErrorRateOfBloomFilter=20
    filterDataCleanTimeSpan=86400000
    filterSupportRetry=false
    enablePropertyFilter=false
    compressedRegister=false
    forceRegister=true
    registerNameServerPeriod=30000
    transactionTimeOut=6000
    transactionCheckMax=15
    transactionCheckInterval=60000
    #Broker 对外服务的监听端口
    listenPort=10911
    serverWorkerThreads=8
    serverCallbackExecutorThreads=0
    serverSelectorThreads=3
    serverOnewaySemaphoreValue=256
    serverAsyncSemaphoreValue=64
    serverChannelMaxIdleTimeSeconds=120
    serverSocketSndBufSize=131072
    serverSocketRcvBufSize=131072
    serverPooledByteBufAllocatorEnable=true
    useEpollNativeSelector=false
    clientWorkerThreads=4
    clientCallbackExecutorThreads=2
    clientOnewaySemaphoreValue=65535
    clientAsyncSemaphoreValue=65535
    connectTimeoutMillis=3000
    channelNotActiveInterval=60000
    clientChannelMaxIdleTimeSeconds=120
    clientSocketSndBufSize=131072
    clientSocketRcvBufSize=131072
    clientPooledByteBufAllocatorEnable=false
    clientCloseSocketIfTimeout=false
    useTLS=false
    storePathRootDir=/data/rocketmq/store
    storePathCommitLog=/root/rocketmq/store/commitlog
    #commitLog每个文件的大小默认1G
    mapedFileSizeCommitLog=1073741824
    #ConsumeQueue每个文件默认存60W条,根据业务情况调整,
    mapedFileSizeConsumeQueue=3000000
    enableConsumeQueueExt=false
    mappedFileSizeConsumeQueueExt=50331648
    bitMapLengthConsumeQueueExt=64
    flushIntervalCommitLog=500
    commitIntervalCommitLog=200
    useReentrantLockWhenPutMessage=false
    flushCommitLogTimed=false
    flushIntervalConsumeQueue=1000
    cleanResourceInterval=10000
    deleteCommitLogFilesInterval=100
    deleteConsumeQueueFilesInterval=100
    destroyMapedFileIntervalForcibly=120000
    redeleteHangedFileInterval=120000
    #删除文件时间点,默认凌晨 2点
    deleteWhen=02
    #检测物理文件磁盘空间
    diskMaxUsedSpaceRatio=75
    #文件保留时间,默认 72 小时
    fileReservedTime=128
    putMsgIndexHightWater=600000
    #限制的消息大小
    maxMessageSize=65536
    checkCRCOnRecover=true
    flushCommitLogLeastPages=4
    commitCommitLogLeastPages=4
    flushLeastPagesWhenWarmMapedFile=4096
    flushConsumeQueueLeastPages=2
    flushCommitLogThoroughInterval=10000
    commitCommitLogThoroughInterval=200
    flushConsumeQueueThoroughInterval=60000
    maxTransferBytesOnMessageInMemory=262144
    maxTransferCountOnMessageInMemory=32
    maxTransferBytesOnMessageInDisk=65536
    maxTransferCountOnMessageInDisk=8
    accessMessageInMemoryMaxRatio=40
    messageIndexEnable=true
    maxHashSlotNum=5000000
    maxIndexNum=20000000
    maxMsgsNumBatch=64
    messageIndexSafe=false
    haListenPort=10912
    haSendHeartbeatInterval=5000
    haHousekeepingInterval=20000
    haTransferBatchSize=32768
    haMasterAddress=
    haSlaveFallbehindMax=268435456
    brokerRole=ASYNC_MASTER
    flushDiskType=ASYNC_FLUSH
    syncFlushTimeout=5000
    messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h
    flushDelayOffsetInterval=10000
    cleanFileForciblyEnable=true
    warmMapedFileEnable=false
    offsetCheckInSlave=false
    debugLockEnable=false
    duplicationEnable=false
    diskFallRecorded=true
    osPageCacheBusyTimeOutMills=1000
    defaultQueryMaxNum=32
    transientStorePoolEnable=false
    transientStorePoolSize=5
    fastFailIfNoBufferInStorePool=false

    修改配置文件,以 vim /data/rocketmq/conf/2m-noslave/broker-b.properties 为例

    rocketmqHome=/data/rocketmq
    # 如果是一个nameserver,直接写一个地址即可
    namesrvAddr=172.16.230.21:9876;172.16.230.22:9876;172.16.230.23:9876
    brokerName=borker-b
    brokerClusterName=DefaultCluster
    brokerId=0
    brokerPermission=6
    #在发送消息时,自动创建服务器不存在的topic,默认创建的队列数
    defaultTopicQueueNums=4
    #是否允许 Broker 自动创建Topic,建议线下开启,线上关闭
    autoCreateTopicEnable=true
    clusterTopicEnable=true
    brokerTopicEnable=true
    #是否允许 Broker 自动创建订阅组,建议线下开启,线上关闭
    autoCreateSubscriptionGroup=true
    sendMessageThreadPoolNums=1
    pullMessageThreadPoolNums=20
    queryMessageThreadPoolNums=10
    adminBrokerThreadPoolNums=16
    clientManageThreadPoolNums=32
    consumerManageThreadPoolNums=32
    heartbeatThreadPoolNums=2
    endTransactionThreadPoolNums=12
    flushConsumerOffsetInterval=5000
    flushConsumerOffsetHistoryInterval=60000
    rejectTransactionMessage=false
    fetchNamesrvAddrByAddressServer=false
    sendThreadPoolQueueCapacity=10000
    pullThreadPoolQueueCapacity=100000
    queryThreadPoolQueueCapacity=20000
    clientManagerThreadPoolQueueCapacity=1000000
    consumerManagerThreadPoolQueueCapacity=1000000
    heartbeatThreadPoolQueueCapacity=50000
    endTransactionPoolQueueCapacity=100000
    filterServerNums=0
    longPollingEnable=true
    shortPollingTimeMills=1000
    notifyConsumerIdsChangedEnable=true
    highSpeedMode=false
    commercialEnable=true
    commercialTimerCount=1
    commercialTransCount=1
    commercialBigCount=1
    commercialBaseCount=1
    transferMsgByHeap=true
    maxDelayTime=40
    regionId=DefaultRegion
    registerBrokerTimeoutMills=6000
    slaveReadEnable=false
    disableConsumeIfConsumerReadSlowly=false
    consumerFallbehindThreshold=17179869184
    brokerFastFailureEnable=true
    waitTimeMillsInSendQueue=200
    waitTimeMillsInPullQueue=5000
    waitTimeMillsInHeartbeatQueue=31000
    waitTimeMillsInTransactionQueue=3000
    startAcceptSendRequestTimeStamp=0
    traceOn=true
    enableCalcFilterBitMap=false
    expectConsumerNumUseFilter=32
    maxErrorRateOfBloomFilter=20
    filterDataCleanTimeSpan=86400000
    filterSupportRetry=false
    enablePropertyFilter=false
    compressedRegister=false
    forceRegister=true
    registerNameServerPeriod=30000
    transactionTimeOut=6000
    transactionCheckMax=15
    transactionCheckInterval=60000
    #Broker 对外服务的监听端口
    listenPort=10911
    serverWorkerThreads=8
    serverCallbackExecutorThreads=0
    serverSelectorThreads=3
    serverOnewaySemaphoreValue=256
    serverAsyncSemaphoreValue=64
    serverChannelMaxIdleTimeSeconds=120
    serverSocketSndBufSize=131072
    serverSocketRcvBufSize=131072
    serverPooledByteBufAllocatorEnable=true
    useEpollNativeSelector=false
    clientWorkerThreads=4
    clientCallbackExecutorThreads=2
    clientOnewaySemaphoreValue=65535
    clientAsyncSemaphoreValue=65535
    connectTimeoutMillis=3000
    channelNotActiveInterval=60000
    clientChannelMaxIdleTimeSeconds=120
    clientSocketSndBufSize=131072
    clientSocketRcvBufSize=131072
    clientPooledByteBufAllocatorEnable=false
    clientCloseSocketIfTimeout=false
    useTLS=false
    storePathRootDir=/data/rocketmq/store
    storePathCommitLog=/root/rocketmq/store/commitlog
    #commitLog每个文件的大小默认1G
    mapedFileSizeCommitLog=1073741824
    #ConsumeQueue每个文件默认存60W条,根据业务情况调整,
    mapedFileSizeConsumeQueue=3000000
    enableConsumeQueueExt=false
    mappedFileSizeConsumeQueueExt=50331648
    bitMapLengthConsumeQueueExt=64
    flushIntervalCommitLog=500
    commitIntervalCommitLog=200
    useReentrantLockWhenPutMessage=false
    flushCommitLogTimed=false
    flushIntervalConsumeQueue=1000
    cleanResourceInterval=10000
    deleteCommitLogFilesInterval=100
    deleteConsumeQueueFilesInterval=100
    destroyMapedFileIntervalForcibly=120000
    redeleteHangedFileInterval=120000
    #删除文件时间点,默认凌晨 2点
    deleteWhen=02
    #检测物理文件磁盘空间
    diskMaxUsedSpaceRatio=75
    #文件保留时间,默认 72 小时
    fileReservedTime=128
    putMsgIndexHightWater=600000
    #限制的消息大小
    maxMessageSize=65536
    checkCRCOnRecover=true
    flushCommitLogLeastPages=4
    commitCommitLogLeastPages=4
    flushLeastPagesWhenWarmMapedFile=4096
    flushConsumeQueueLeastPages=2
    flushCommitLogThoroughInterval=10000
    commitCommitLogThoroughInterval=200
    flushConsumeQueueThoroughInterval=60000
    maxTransferBytesOnMessageInMemory=262144
    maxTransferCountOnMessageInMemory=32
    maxTransferBytesOnMessageInDisk=65536
    maxTransferCountOnMessageInDisk=8
    accessMessageInMemoryMaxRatio=40
    messageIndexEnable=true
    maxHashSlotNum=5000000
    maxIndexNum=20000000
    maxMsgsNumBatch=64
    messageIndexSafe=false
    haListenPort=10912
    haSendHeartbeatInterval=5000
    haHousekeepingInterval=20000
    haTransferBatchSize=32768
    haMasterAddress=
    haSlaveFallbehindMax=268435456
    brokerRole=ASYNC_MASTER
    flushDiskType=ASYNC_FLUSH
    syncFlushTimeout=5000
    messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h
    flushDelayOffsetInterval=10000
    cleanFileForciblyEnable=true
    warmMapedFileEnable=false
    offsetCheckInSlave=false
    debugLockEnable=false
    duplicationEnable=false
    diskFallRecorded=true
    osPageCacheBusyTimeOutMills=1000
    defaultQueryMaxNum=32
    transientStorePoolEnable=false
    transientStorePoolSize=5
    fastFailIfNoBufferInStorePool=false

    ####  如果要实现双主双从#########

     

    ################################################

    修改日志保存路径 /data/rocketmq/conf/ 中的xml文件

    sed -i 's#${user.home}#/data/rocketmq/logs#g' *.xml

    新建日志目录,消息存储路径

    [root@rocketmq1 conf]#mkdir -p /data/rocketmq/store
    [root@rocketmq1 conf]#mkdir -p /data/rocketmq/store/commitlog
    [root@rocketmq1 conf]#mkdir -p /data/rocketmq/logs 

    修改nameserver初始堆参数

    [root@rocketmq1 conf]# vim /data/rocketmq/bin/runserver.sh

    修改broker 初始化堆参数

     [root@rocketmq1 conf]# vim /data/rocketmq/bin/runbroker.sh

    启动nameserver 和 broker进程

    [root@rocketmq1 conf]# cd /data/rocketmq/bin/
    
    #启动nameserver
    [root@rocketmq1 bin]# nohup ./mqnamesrv &
    
    查看监听端口
    [root@rocketmq1 bin]# netstat -anpt | grep java
    tcp6       0      0 :::9876                 :::*                    LISTEN      11086/java  
    
    查看日志:
    [root@rocketmq1 bin]# tail -f /data/rocketmq/logs/rocketmqlogs/namesrv.log 
    2018-12-20 16:49:13 INFO main - The Name Server boot success. serializeType=JSON
    
    #启动 broker
    [root@rocketmq1 bin]# nohup ./mqbroker -c /data/rocketmq/conf/2m-noslave/broker-a.properties >/dev/null 2>&1 &
    
    监听10911端口
    [root@rocketmq1 bin]# netstat -anpt | grep java
    tcp6       0      0 :::9876                 :::*                    LISTEN      11086/java          
    tcp6       0      0 :::10909                :::*                    LISTEN      11230/java          
    tcp6       0      0 :::10911                :::*                    LISTEN      11230/java          
    tcp6       0      0 :::10912                :::*                    LISTEN      11230/java          
    tcp6       0      0 172.16.230.21:9876      172.16.230.21:47064     ESTABLISHED 11086/java          
    tcp6       0      0 172.16.230.21:47064     172.16.230.21:9876      ESTABLISHED 11230/java

    查看集群状态

    [root@rocketmq1 bin]# sh mqadmin clusterList -n "172.16.230.21:9876";
    Java HotSpot(TM) 64-Bit Server VM warning: ignoring option PermSize=128m; support was removed in 8.0
    Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=128m; support was removed in 8.0
    #Cluster Name     #Broker Name            #BID  #Addr                  #Version                #InTPS(LOAD)       #OutTPS(LOAD) #PCWait(ms) #Hour #SPACE
    DefaultCluster    borker-a                0     172.16.230.21:10911    V4_3_2                   0.00(0,0ms)         0.00(0,0ms)          0 429248.92 -1.0000

    测试,创建topic

    ./mqadmin updateTopic  -n 172.16.230.21:9876 -b localhost:10911 -t tx-mq-TOPIC

    rocketmq 集群监控

    git clone https://github.com/apache/incubator-rocketmq-externals/tree/master/rocketmq-console

    编译 rocketmq-console jar包

    [root@localhost data]# ls
    jdk  maven  rocketmq-console-ng-1.0.0.jar  rocketmq-externals
    [root@localhost data]# cd rocketmq-externals/
    [root@localhost rocketmq-externals]# ls
    dev        rocketmq-console  rocketmq-flink  rocketmq-go     rocketmq-iot-bridge  rocketmq-mysql  rocketmq-python  rocketmq-sentinel    rocketmq-spark
    README.md  rocketmq-docker   rocketmq-flume  rocketmq-hbase  rocketmq-jms         rocketmq-php    rocketmq-redis   rocketmq-serializer
    
    [root@localhost rocketmq
    -externals]# cd rocketmq-console/

    #修改nameserver 地址
    [root@localhost rocketmq-externals]# vim src/main/resources/application.properties/
    
    
    
    
    
    [root@localhost rocketmq-console]# ls
    doc  LICENSE  NOTICE  pom.xml  README.md  src  style  
    [root@localhost rocketmq-console]# mvn clear package -Dmaven.test.skip=true
    [root@localhost rocketmq-console]# cp target/rocketmq-console-ng-1.0.0.jar   /data
    [root@localhost rocketmq-console]#  cd /data/
    [root@localhost rocketmq-console]#  jar -jar  rocketmq-console-ng-1.0.0.jar


    压力测试:

    [root@harbor benchmark]# export NAMESRV_ADDR=localhost:9876
    [root@harbor benchmark]# cd /opt/senyint/rocketmq-all/benchmark
    [root@harbor benchmark]# ./producer.sh

  • 相关阅读:
    《演讲之禅》助你成长为一名合格程序员
    JavaScript中设置元素class的三种方法小结
    数据类型和Json格式
    JavaScript给onclick事件赋值,动态传参数实例辩错
    JSON.parse()和JSON.stringify()使用
    javascript写在<head>和<body>里的区别
    Source Insight 相关
    串口接收字符时,最后一个字符延时发送
    嵌入式编程中调试问题全记录
    新版数采仪开发问题全记录
  • 原文地址:https://www.cnblogs.com/fengjian2016/p/10150394.html
Copyright © 2011-2022 走看看