4、MongoDB分片集群

# 分别在四台节点配置YUM源并创建数据目录
[root@node01 ~]# cat > /etc/yum.repos.d/mongodb.repo << EOF
[mongodb]
name=MongoDB Repository
baseurl=https://mirrors.tuna.tsinghua.edu.cn/mongodb/yum/el7/
gpgcheck=0
enabled=1
EOF
[root@node01 ~]# yum clean all
[root@node01 ~]# yum makecache
---
# 在node2节点的Config Server
[root@node02 ~]# yum install -y mongodb-org
# 为三台ConfigServer提供配置
[root@node02 ~]# cp /etc/mongod.conf /etc/mongod1.conf 
[root@node02 ~]# cp /etc/mongod.conf /etc/mongod2.conf    
[root@node02 ~]# cp /etc/mongod.conf /etc/mongod3.conf 
# 为三台ConfigServer配置数据目录
[root@node02 ~]# mkdir -p /data/mongodb/configsvr{1..3}/{data,logs,run}
[root@node02 ~]# tree /data/
/data/
└── mongodb
    ├── configsvr1
    │   ├── data
    │   ├── logs
    │   └── run
    ├── configsvr2
    │   ├── data
    │   ├── logs
    │   └── run
    └── configsvr3
        ├── data
        ├── logs
        └── run
# 配置第一台ConfigServer
[root@node02 ~]# grep '^[^#]' /etc/mongod1.conf 
systemLog:
  destination: file
  logAppend: true
  path: /data/mongodb/configsvr1/logs/mongod.log
storage:
  dbPath: /data/mongodb/configsvr1/data
  journal:
    enabled: true
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /data/mongodb/configsvr1/run/mongod.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 27027
  bindIp: 0.0.0.0  # Enter 0.0.0.0,:: to bind to all IPv4 and IPv6 addresses or, alternatively, use the net.bindIpAll setting.
replication:
  replSetName: csvr
sharding:
  clusterRole: configsvr
# 配置第二台ConfigServer提供配置数据目录
[root@node02 ~]# grep '^[^#]' /etc/mongod2.conf 
systemLog:
  destination: file
  logAppend: true
  path: /data/mongodb/configsvr2/logs/mongod.log
storage:
  dbPath: /data/mongodb/configsvr2/data
  journal:
    enabled: true
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /data/mongodb/configsvr2/run/mongod.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 27028
  bindIp: 0.0.0.0  # Enter 0.0.0.0,:: to bind to all IPv4 and IPv6 addresses or, alternatively, use the net.bindIpAll setting.
replication:
  replSetName: csvr
sharding:
  clusterRole: configsvr
# 配置第三台ConfigServer提供配置数据目录
[root@node02 ~]# grep '^[^#]' /etc/mongod3.conf 
systemLog:
  destination: file
  logAppend: true
  path: /data/mongodb/configsvr3/logs/mongod.log
storage:
  dbPath: /data/mongodb/configsvr3/data
  journal:
    enabled: true
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /data/mongodb/configsvr3/run/mongod.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 27029
  bindIp: 0.0.0.0  # Enter 0.0.0.0,:: to bind to all IPv4 and IPv6 addresses or, alternatively, use the net.bindIpAll setting.
replication:
  replSetName: csvr
sharding:
  clusterRole: configsvr
# 启动三台ConfigServer
[root@node02 ~]# mongod -f /etc/mongod1.conf 
[root@node02 ~]# mongod -f /etc/mongod2.conf 
[root@node02 ~]# mongod -f /etc/mongod3.conf 
[root@node02 ~]# netstat -ntlp|grep '2702'
tcp        0      0 0.0.0.0:27027           0.0.0.0:*               LISTEN      7046/mongod         
tcp        0      0 0.0.0.0:27028           0.0.0.0:*               LISTEN      7151/mongod         
tcp        0      0 0.0.0.0:27029           0.0.0.0:*               LISTEN      7231/mongod
# 初始化ConfigServer副本集
[root@node02 ~]# mongo --port 27027
> rs.initiate()
csvr:PRIMARY> rs.add('node02.cce.com:27028')
csvr:PRIMARY> rs.add('node02.cce.com:27029')
# 查看ConfigServer副本集
---
# 在node01上安装mongos
[root@node01 ~]# yum install -y mongodb-org-mongos mongodb-org-shell
# 创建mongos的日志目录
[root@node01 ~]# mkdir /data/mongodb/logs
[root@node01 ~]# tree /data/
/data/
└── mongodb
    └── logs
# 以命令行的方式启动mongos，启动mongos只需要指定config server所在地址即可，可以是逗号隔开的三个或者一个
# [root@node01 ~]# mongos --configdb csvr/172.16.1.2:27027,172.16.1.2:27028,172.16.1.2:27029 --fork
# 提供mongos的配置文件
[root@node01 ~]# cat > /etc/mongos.conf << EOF
systemLog:
  destination: file
  path: /data/mongodb/logs/mongos.log
  logAppend: true
net:
  bindIp: 0.0.0.0
  port: 27017
sharding:
  configDB: csvr/172.16.1.2:27027,172.16.1.2:27028,172.16.1.2:27029
processManagement:
  fork: true
EOF
# 启动mongos
[root@node01 ~]# mongos -f /etc/mongos.conf 
[root@node01 ~]# netstat -ntlp|grep '270'
tcp        0      0 0.0.0.0:27017           0.0.0.0:*               LISTEN      25803/mongos    
---
# 在node03上创建一个shard副本集，首先创建数据目录
[root@node03 ~]# mkdir -pv /data/mongod{1..3}/{data,logs,run}
[root@node03 ~]# tree /data/
/data/
├── mongod1
│   ├── data
│   ├── logs
│   └── run
├── mongod2
│   ├── data
│   ├── logs
│   └── run
└── mongod3
    ├── data
    ├── logs
    └── run
# 安装mongodb
[root@node03 ~]# yum install -y mongodb-org
# 提供三台mongod的配置文件
[root@node03 ~]# cp /etc/mongod.conf /etc/mongod1.conf     
[root@node03 ~]# cp /etc/mongod.conf /etc/mongod2.conf
[root@node03 ~]# cp /etc/mongod.conf /etc/mongod3.conf
# 配置第一台副本集节点的mongod
[root@node03 ~]# grep '^[^#]' /etc/mongod1.conf
systemLog:
  destination: file
  logAppend: true
  path: /data/mongod1/logs/mongod.log
storage:
  dbPath: /data/mongod1/data
  journal:
    enabled: true
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /data/mongod1/run/mongod.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 27017
  bindIp: 0.0.0.0  # Enter 0.0.0.0,:: to bind to all IPv4 and IPv6 addresses or, alternatively, use the net.bindIpAll setting.
replication:
  replSetName: shard1
sharding:
  clusterRole: shardsvr
# 配置第二台副本集节点的mongod
[root@node03 ~]# grep '^[^#]' /etc/mongod2.conf
systemLog:
  destination: file
  logAppend: true
  path: /data/mongod2/logs/mongod.log
storage:
  dbPath: /data/mongod2/data
  journal:
    enabled: true
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /data/mongod2/run/mongod.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 27018
  bindIp: 0.0.0.0  # Enter 0.0.0.0,:: to bind to all IPv4 and IPv6 addresses or, alternatively, use the net.bindIpAll setting.
replication:
  replSetName: shard1
sharding:
  clusterRole: shardsvr
# 配置第三台副本集节点的mongod
[root@node03 ~]# grep '^[^#]' /etc/mongod3.conf
systemLog:
  destination: file
  logAppend: true
  path: /data/mongod3/logs/mongod.log
storage:
  dbPath: /data/mongod3/data
  journal:
    enabled: true
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /data/mongod3/run/mongod.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 27019
  bindIp: 0.0.0.0  # Enter 0.0.0.0,:: to bind to all IPv4 and IPv6 addresses or, alternatively, use the net.bindIpAll setting.
replication:
  replSetName: shard1
sharding:
  clusterRole: shardsvr
# 启动三台mongod
[root@node03 ~]# mongod -f /etc/mongod1.conf 
[root@node03 ~]# mongod -f /etc/mongod2.conf  
[root@node03 ~]# mongod -f /etc/mongod3.conf 
# 初始化副本集
[root@node03 ~]# mongo
> rs.initiate()
shard1:PRIMARY> rs.add('node03.cce.com:27018')
shard1:PRIMARY> rs.add('node03.cce.com:27019')
---
# 在node04上创建一个shard副本集，首先创建数据目录
[root@node04 ~]# mkdir -pv /data/mongod{1..3}/{data,logs,run}
[root@node04 ~]# tree /data/
/data/
├── mongod1
│   ├── data
│   ├── logs
│   └── run
├── mongod2
│   ├── data
│   ├── logs
│   └── run
└── mongod3
    ├── data
    ├── logs
    └── run
# 安装mongodb
[root@node04 ~]# yum install -y mongodb-org
# 提供三台mongod的配置文件
[root@node04 ~]# cp /etc/mongod.conf /etc/mongod1.conf     
[root@node04 ~]# cp /etc/mongod.conf /etc/mongod2.conf
[root@node04 ~]# cp /etc/mongod.conf /etc/mongod3.conf
# 配置第一台副本集节点的mongod
[root@node04 ~]# grep '^[^#]' /etc/mongod1.conf
systemLog:
  destination: file
  logAppend: true
  path: /data/mongod1/logs/mongod.log
storage:
  dbPath: /data/mongod1/data
  journal:
    enabled: true
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /data/mongod1/run/mongod.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 27017
  bindIp: 0.0.0.0  # Enter 0.0.0.0,:: to bind to all IPv4 and IPv6 addresses or, alternatively, use the net.bindIpAll setting.
replication:
  replSetName: shard2
sharding:
  clusterRole: shardsvr
# 配置第二台副本集节点的mongod
[root@node04 ~]# grep '^[^#]' /etc/mongod2.conf
systemLog:
  destination: file
  logAppend: true
  path: /data/mongod2/logs/mongod.log
storage:
  dbPath: /data/mongod2/data
  journal:
    enabled: true
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /data/mongod2/run/mongod.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 27018
  bindIp: 0.0.0.0  # Enter 0.0.0.0,:: to bind to all IPv4 and IPv6 addresses or, alternatively, use the net.bindIpAll setting.
replication:
  replSetName: shard2
sharding:
  clusterRole: shardsvr
# 配置第三台副本集节点的mongod
[root@node04 ~]# grep '^[^#]' /etc/mongod3.conf
systemLog:
  destination: file
  logAppend: true
  path: /data/mongod3/logs/mongod.log
storage:
  dbPath: /data/mongod3/data
  journal:
    enabled: true
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /data/mongod3/run/mongod.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 27019
  bindIp: 0.0.0.0  # Enter 0.0.0.0,:: to bind to all IPv4 and IPv6 addresses or, alternatively, use the net.bindIpAll setting.
replication:
  replSetName: shard2
sharding:
  clusterRole: shardsvr
# 启动三台mongod
[root@node04 ~]# mongod -f /etc/mongod1.conf 
[root@node04 ~]# mongod -f /etc/mongod2.conf  
[root@node04 ~]# mongod -f /etc/mongod3.conf 
# 初始化副本集
[root@node04 ~]# mongo
> rs.initiate()
shard2:PRIMARY> rs.add("node04.cce.com:27018")
shard2:PRIMARY> rs.add("node04.cce.com:27019")
---# 进入mongos将node03和node04的副本集作为分片节点加入分片集群
[root@node01 ~]# mongo
mongos> sh.addShard("shard1/node03.cce.com:27017,node03.cce.com:27018,node03.cce.com:27019")
mongos> sh.addShard("shard2/node04.cce.com:27017,node04.cce.com:27018,node04.cce.com:27019")
# 列出我们有多少个shards
mongos> db.runCommand('listShards')
{
        "shards" : [
                {
                        "_id" : "shard1",
                        "host" : "shard1/node03.cce.com:27017,node03.cce.com:27018,node03.cce.com:27019",
                        "state" : 1
                },
                {
                        "_id" : "shard2",
                        "host" : "shard2/node04.cce.com:27017,node04.cce.com:27018,node04.cce.com:27019",
                        "state" : 1
                }
        ],
        "ok" : 1,
        "operationTime" : Timestamp(1590473411, 3),
        "$clusterTime" : {
                "clusterTime" : Timestamp(1590473411, 3),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        }
}
mongos> sh.status()
--- Sharding Status --- 
  sharding version: {
        "_id" : 1,
        "minCompatibleVersion" : 5,
        "currentVersion" : 6,
        "clusterId" : ObjectId("5ecc799016a6e3f07938468d")
  }
  shards:   # Shard分片节点
        {  "_id" : "shard1",  "host" : "shard1/node03.cce.com:27017,node03.cce.com:27018,node03.cce.com:27019",  "state" : 1 }
        {  "_id" : "shard2",  "host" : "shard2/node04.cce.com:27017,node04.cce.com:27018,node04.cce.com:27019",  "state" : 1 }
...
  databases:  # 支持分片的数据库，primary表示主shard，一个库内并非所有的collection都需要分片的，那对于那些不分片的就放在了主shard上，而那些分片可能在各shard上都有数据，partitioned表示是否支撑shard
        {  "_id" : "config",  "primary" : "config",  "partitioned" : true }
# 接下来就开始指定数据库分片了，首先创建一个database和数据库和collection
mongos> use cce;
mongos> sh.enableSharding('cce')  # 启用cce库分片功能，让cce库支持分片
mongos> sh.shardCollection('cce.user',{age: 1})  # 指定在cce库的user collection上面创建索引，并且以age为索引的键，这样我们的cce.user就支持分片了
mongos> sh.status()
...
                        主shard                        是否支持分片
{  "_id" : "cce",  "primary" : "shard1",  "partitioned" : true,  "version" : {  "uuid" : UUID("66464020-cc8f-464b-9483-112a4343fd7a"),  "lastMod" : 1 } }
                cce.user
                        shard key: { "age" : 1 } # 分片索引键，升序索引
                        unique: false # 是否唯一，那就表示一个年龄只有一个人
                        balancing: true 
                        chunks:  # 第一个分片放到哪里了，下面表示第一个分片放在了shard1
                                shard1  1
                        { "age" : { "$minKey" : 1 } } -->> { "age" : { "$maxKey" : 1 } } on : shard1 Timestamp(1, 0)
...
# 为了模拟测试，设置chunk大小，默认单个chunk为64M，如果达不到不会进去分片，此处设置成1M
mongos> use config;
mongos> db.settings.insert({"_id":"chunksize","value":1});
# 创建十万条测试数据
mongos> for (i=1;i<=100000;i++) db.user.insert({name: 'cce'+i,age: (i%120),email: ('mail'+i)+'@163.com',address: 'https://blog.doorta.com'})
# 在replSet1上查看一共有多少条数据
[root@node03 ~]# mongo
shard1:PRIMARY> use cce
shard1:PRIMARY> db.user.count()
55818
# 在replSet2上查看一共有多少条数据
[root@node04 ~]# mongo
shard2:PRIMARY> use cce
shard2:PRIMARY> db.user.count()
44182
# 查看分片规则
mongos> sh.status()
databases:
        {  "_id" : "cce",  "primary" : "shard1",  "partitioned" : true,  "version" : {  "uuid" : UUID("66464020-cc8f-464b-9483-112a4343fd7a"),  "lastMod" : 1 } }
                cce.user
                        shard key: { "age" : 1 }
                        unique: false
                        balancing: true
                        chunks:  # 当分片差距太大的时候，会自动均衡
                                shard1  9
                                shard2  10
                        { "age" : { "$minKey" : 1 } } -->> { "age" : 0 } on : shard1 Timestamp(10, 0) 
                        { "age" : 0 } -->> { "age" : 8 } on : shard1 Timestamp(11, 0) 
                        { "age" : 8 } -->> { "age" : 16 } on : shard2 Timestamp(9, 2) 
                        { "age" : 16 } -->> { "age" : 18 } on : shard2 Timestamp(9, 3) 
                        { "age" : 18 } -->> { "age" : 26 } on : shard2 Timestamp(8, 2) 
                        { "age" : 26 } -->> { "age" : 34 } on : shard2 Timestamp(8, 3) 
                        { "age" : 34 } -->> { "age" : 36 } on : shard2 Timestamp(8, 4) 
                        { "age" : 36 } -->> { "age" : 41 } on : shard2 Timestamp(10, 1) 
                        { "age" : 41 } -->> { "age" : 49 } on : shard2 Timestamp(11, 1) 
                        { "age" : 49 } -->> { "age" : 58 } on : shard2 Timestamp(7, 9) 
                        { "age" : 58 } -->> { "age" : 60 } on : shard2 Timestamp(7, 10) 
                        { "age" : 60 } -->> { "age" : 68 } on : shard1 Timestamp(7, 5) 
                        { "age" : 68 } -->> { "age" : 77 } on : shard1 Timestamp(7, 6) 
                        { "age" : 77 } -->> { "age" : 79 } on : shard1 Timestamp(7, 7) 
                        { "age" : 79 } -->> { "age" : 83 } on : shard1 Timestamp(8, 1) 
                        { "age" : 83 } -->> { "age" : 100 } on : shard1 Timestamp(7, 1) 
                        { "age" : 100 } -->> { "age" : 108 } on : shard1 Timestamp(7, 11) 
                        { "age" : 108 } -->> { "age" : 119 } on : shard1 Timestamp(7, 12) 
                        { "age" : 119 } -->> { "age" : { "$maxKey" : 1 } } on : shard2 Timestamp(8, 0) 
# 查看均衡器是否正在运行，如果此时没有出现chunk不均衡的情况，那么你会看到均衡器是没有启动的，如果出现了不均衡，它默认会自动启用的；
mongos> sh.isBalancerRunning()

admin > sh.shardCollection( "数据库名.集合名", { 片键: "hashed" } )
admin> db.user.ensureIndex( { age: "hashed" } )
admin > sh.shardCollection( "cce.user", {age: "hashed" } )

# 判断是否Shard集群
admin> db.runCommand({ isdbgrid : 1})
# 列出所有分片信息
admin> db.runCommand({ listshards : 1})
# 列出开启分片的数据库
admin> use config
config> db.databases.find( { "partitioned": true } )
config> db.databases.find() //列出所有数据库分片情况
# 查看分片的片键
config> db.collections.find()
# 查看分片的详细信息
admin> db.printShardingStatus()
# 或
admin> sh.status()
# 删除分片节点
> sh.getBalancerState()
mongos> db.runCommand( { removeShard: "shard2" } )
# 查看mongo集群是否开启了 balance 状态
mongos> sh.getBalancerState()
# mongodb在做自动分片平衡的时候，或引起数据库响应的缓慢，可以通过禁用自动平衡以及设置自动平衡进行的时间来解决这一问题。
> use config
> db.settings.update( { _id: "balancer" }, { $set : { stopped: true } } , true );
# 自定义自动平衡进行的时间段
> use config
> db.settings.update({ _id : "balancer" }, { $set : { activeWindow : { start : "21:00", stop : "9:00" } } }, true )