mongodb 3.4 学习 (四）分片

https://www.linode.com/docs/databases/mongodb/build-database-clusters-with-mongodb

由三部分组成
shard: 每个shard包括一个所有shard数据的子集. 每个shard能够部署成replica set
mongos: 做为查询路由, 提供clientt与sharded cluster的接口
config servers: 存储群集的metadata和配置，必须部署为replica set

# 生产环境部署
kvm-70-102 10.0.70.102 # mongos
kvm-70-103 10.0.70.103 # config
kvm-70-102 10.0.70.104 # shard1
kvm-70-103 10.0.70.105 # shard2

config

# 建立数据目录
mkdir -p /opt/mongo_{27021,27022,27023}

# 配置文件
cat > /etc/mongod_27021.conf << EOF
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod_27021.log

storage:
  dbPath: /opt/mongo_27021
  journal:
    enabled: true

processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/mongod_27021.pid

net:
  port: 27021

security:
  keyFile: /etc/mongodb_product_key

replication:
   oplogSizeMB: 2048
   replSetName: config

sharding:
  clusterRole: configsvr
EOF

# 配置文件
cat > /etc/mongod_27022.conf << EOF
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod_27022.log

storage:
  dbPath: /opt/mongo_27022
  journal:
    enabled: true

processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/mongod_27022.pid

net:
  port: 27022

security:
  keyFile: /etc/mongodb_product_key

replication:
   oplogSizeMB: 2048
   replSetName: config

sharding:
  clusterRole: configsvr
EOF

# 配置文件
cat > /etc/mongod_27023.conf << EOF
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod_27023.log

storage:
  dbPath: /opt/mongo_27023
  journal:
    enabled: true

processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/mongod_27023.pid

net:
  port: 27023

security:
  keyFile: /etc/mongodb_product_key

replication:
   oplogSizeMB: 2048
   replSetName: config

sharding:
  clusterRole: configsvr
EOF

# 启动复制集
mongod --config /etc/mongod_27021.conf
mongod --config /etc/mongod_27022.conf
mongod --config /etc/mongod_27023.conf

# 配置config
mongo --port 27021

rs.initiate(
  {
    _id: 'config',
    configsvr: true,
    members: [
      { _id : 0, host : '10.0.70.103:27021' },
      { _id : 1, host : '10.0.70.103:27022' },
      { _id : 2, host : '10.0.70.103:27023' }
    ]
  }
)
{ "ok" : 1 }

shard1

# 建立数据目录
mkdir -p /opt/mongo_{27021,27022,27023}

# 配置文件
cat > /etc/mongod_27021.conf << EOF
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod_27021.log

storage:
  dbPath: /opt/mongo_27021
  journal:
    enabled: true

processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/mongod_27021.pid

net:
  port: 27021

security:
  keyFile: /etc/mongodb_product_key

replication:
   oplogSizeMB: 2048
   replSetName: shard1

sharding:
  clusterRole:  shardsvr
EOF

# 配置文件
cat > /etc/mongod_27022.conf << EOF
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod_27022.log

storage:
  dbPath: /opt/mongo_27022
  journal:
    enabled: true

processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/mongod_27022.pid

net:
  port: 27022

security:
  keyFile: /etc/mongodb_product_key

replication:
   oplogSizeMB: 2048
   replSetName: shard1

sharding:
  clusterRole: shardsvr
EOF

# 配置文件
cat > /etc/mongod_27023.conf << EOF
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod_27023.log

storage:
  dbPath: /opt/mongo_27023
  journal:
    enabled: true

processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/mongod_27023.pid

net:
  port: 27023

security:
  keyFile: /etc/mongodb_product_key

replication:
   oplogSizeMB: 2048
   replSetName: shard1

sharding:
  clusterRole: shardsvr
EOF

# 启动复制集
mongod --config /etc/mongod_27021.conf
mongod --config /etc/mongod_27022.conf
mongod --config /etc/mongod_27023.conf

# 配置config
mongo --port 27021

rs.initiate(
  {
    _id: 'shard1',
    members: [
      { _id : 0, host : '10.0.70.104:27021' },
      { _id : 1, host : '10.0.70.104:27022' },
      { _id : 2, host : '10.0.70.104:27023' }
    ]
  }
)
{ "ok" : 1 }

shard2

# 建立数据目录
mkdir -p /opt/mongo_{27021,27022,27023}

# 配置文件
cat > /etc/mongod_27021.conf << EOF
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod_27021.log

storage:
  dbPath: /opt/mongo_27021
  journal:
    enabled: true

processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/mongod_27021.pid

net:
  port: 27021

security:
  keyFile: /etc/mongodb_product_key

replication:
   oplogSizeMB: 2048
   replSetName: shard2

sharding:
  clusterRole:  shardsvr
EOF

# 配置文件
cat > /etc/mongod_27022.conf << EOF
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod_27022.log

storage:
  dbPath: /opt/mongo_27022
  journal:
    enabled: true

processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/mongod_27022.pid

net:
  port: 27022

security:
  keyFile: /etc/mongodb_product_key

replication:
   oplogSizeMB: 2048
   replSetName: shard2

sharding:
  clusterRole: shardsvr
EOF

# 配置文件
cat > /etc/mongod_27023.conf << EOF
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod_27023.log

storage:
  dbPath: /opt/mongo_27023
  journal:
    enabled: true

processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/mongod_27023.pid

net:
  port: 27023

security:
  keyFile: /etc/mongodb_product_key

replication:
   oplogSizeMB: 2048
   replSetName: shard2

sharding:
  clusterRole: shardsvr
EOF

# 启动复制集
mongod --config /etc/mongod_27021.conf
mongod --config /etc/mongod_27022.conf
mongod --config /etc/mongod_27023.conf

# 配置config
mongo --port 27021

rs.initiate(
  {
    _id: 'shard2',
    members: [
      { _id : 0, host : '10.0.70.105:27021' },
      { _id : 1, host : '10.0.70.105:27022' },
      { _id : 2, host : '10.0.70.105:27023' }
    ]
  }
)
{ "ok" : 1 }

mongos

# 配置文件
cat > /etc/mongod_27021.conf << EOF
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod_27021.log

processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/mongod_27021.pid

net:
  port: 27021

security:
  keyFile: /etc/mongodb_product_key

sharding:
  configDB: config/10.0.70.103:27021,10.0.70.103:27022,10.0.70.103:27023
EOF

# 启动mongos
mongos --config /etc/mongod_27021.conf

# 配置
mongo --port 27021

# 添加分片
sh.addShard('shard1/10.0.70.104:27021')
sh.addShard('shard1/10.0.70.104:27022')
sh.addShard('shard1/10.0.70.104:27023')

sh.addShard('shard2/10.0.70.105:27021')
sh.addShard('shard2/10.0.70.105:27022')
sh.addShard('shard2/10.0.70.105:27023')

# 建立超级用户
use admin

db.createUser(
  {
    user: 'admin',
    pwd: '@admin',
    roles: [ { role: 'root', db: 'admin' } ]
  }
);


# 认证登录
db.auth('admin', '@admin')

# app数据库demo表插入100000条数据
use app
for (var i = 1; i < 100000; i++) db.demo.insert({id: i, name: 'ken'})
WriteResult({ "nInserted" : 1 })

# 查看分片状态
sh.status()
--- Sharding Status --- 
  sharding version: {
    "_id" : 1,
    "minCompatibleVersion" : 5,
    "currentVersion" : 6,
    "clusterId" : ObjectId("59261bca37ceff575b36ef09")
}
  shards:
    {  "_id" : "shard1",  "host" : "shard1/10.0.70.104:27021,10.0.70.104:27022,10.0.70.104:27023",  "state" : 1 }
    {  "_id" : "shard2",  "host" : "shard2/10.0.70.105:27021,10.0.70.105:27022,10.0.70.105:27023",  "state" : 1 }
  active mongoses:
    "3.4.4" : 1
 autosplit:
    Currently enabled: yes
  balancer:
    Currently enabled:  yes
    Currently running:  no
        Balancer lock taken at Thu May 25 2017 07:48:42 GMT+0800 (CST) by ConfigServer:Balancer
    Failed balancer rounds in last 5 attempts:  0
    Migration Results for the last 24 hours: 
        No recent migrations
  databases:
    {  "_id" : "app",  "primary" : "shard1",  "partitioned" : false }

# 结果显示没有分区，记录插在shard1服务器

# 配置分片
# 允许分片的数据库
sh.enableSharding('app')

# 为用做shard key的字段建立索引，实际中可以用uuid字段
use app
db.demo.createIndex({id: 1})
{
    "raw" : {
        "shard1/10.0.70.104:27021,10.0.70.104:27022,10.0.70.104:27023" : {
            "createdCollectionAutomatically" : false,
            "numIndexesBefore" : 1,
            "numIndexesAfter" : 2,
            "ok" : 1,
            "$gleStats" : {
                "lastOpTime" : {
                    "ts" : Timestamp(1495675335, 1),
                    "t" : NumberLong(1)
                },
                "electionId" : ObjectId("7fffffff0000000000000001")
            }
        }
    },
    "ok" : 1
}

# 允许分片的collection，指定shard key的字段
sh.shardCollection('app.demo', { id : 'hashed' })
{ "collectionsharded" : "app.demo", "ok" : 1 }


# 插入数据
for (var i = 1; i < 10000; i++) db.demo.insert({id: i, name: 'ken'})
WriteResult({ "nInserted" : 1 })

# 查看状态，可以看到db与collection的分片状态
sh.status()
--- Sharding Status --- 
  sharding version: {
    "_id" : 1,
    "minCompatibleVersion" : 5,
    "currentVersion" : 6,
    "clusterId" : ObjectId("59261bca37ceff575b36ef09")
}
  shards:
    {  "_id" : "shard1",  "host" : "shard1/10.0.70.104:27021,10.0.70.104:27022,10.0.70.104:27023",  "state" : 1 }
    {  "_id" : "shard2",  "host" : "shard2/10.0.70.105:27021,10.0.70.105:27022,10.0.70.105:27023",  "state" : 1 }
  active mongoses:
    "3.4.4" : 1
 autosplit:
    Currently enabled: yes
  balancer:
    Currently enabled:  yes
    Currently running:  no
        Balancer lock taken at Thu May 25 2017 07:48:42 GMT+0800 (CST) by ConfigServer:Balancer
    Failed balancer rounds in last 5 attempts:  0
    Migration Results for the last 24 hours: 
        1 : Success
  databases:
    {  "_id" : "app",  "primary" : "shard1",  "partitioned" : true }
        app.demo
            shard key: { "id" : "hashed" }
            unique: false
            balancing: true
            chunks:
                shard1  2
                shard2  2
            { "id" : { "$minKey" : 1 } } -->> { "id" : NumberLong("-4611686018427387902") } on : shard1 Timestamp(2, 2) 
            { "id" : NumberLong("-4611686018427387902") } -->> { "id" : NumberLong(0) } on : shard1 Timestamp(2, 3) 
            { "id" : NumberLong(0) } -->> { "id" : NumberLong("4611686018427387902") } on : shard2 Timestamp(2, 4) 
            { "id" : NumberLong("4611686018427387902") } -->> { "id" : { "$maxKey" : 1 } } on : shard2 Timestamp(2, 5) 

# shard1
db.demo.find().count()
4992

# shard2
db.demo.find().count()
5007

如果在允许colletion前demo有数据，执行以下命令时会报错。
sh.shardCollection('app.demo', { id : 'hashed' })
{
    "proposedKey" : {
        "id" : "hashed"
    },
    "curIndexes" : [
        {
            "v" : 2,
            "key" : {
                "_id" : 1
            },
            "name" : "_id_",
            "ns" : "app.demo"
        },
        {
            "v" : 2,
            "key" : {
                "id" : 1
            },
            "name" : "id_1",
            "ns" : "app.demo"
        }
    ],
    "ok" : 0,
    "errmsg" : "please create an index that starts with the shard key before sharding."
}

做法是将collection删除，重建索引。

chunksize默认是64M，如果分片过慢的话，可以适当调大，如512M。mongos的配置文件里。

Hashed Sharding

Ranged Sharding