Share my minds: MongoDB replication with arbiter

One of my projects assumed to have database replication.
I'd used standard replication master-slave until last failure.
My master (PRIMARY) went down and slave (SECONDARY) became master (PRIMARY).
When "old" master went back it'd overtaken master from "new" one.
It was unacceptable for me because I ran everything on new master and it contains new data.
Solution was simple - use an arbiter.
It cost one more machine with mongo but with no data on board.

How to set up

1) Create configuration file on first node (master/PRIMARY) with replica set

root@mongo1:~# cat /etc/mongodb1.conf 
dbpath=/var/lib/mongodb1
logpath=/var/log/mongodb/mongodb1.log
logappend=true
bind_ip = mongo1
port = 17017
journal=true
replSet = replication_test

In my case configuration will be almost the same on the other host( except bind_ip and port).

2) Start mongodb on first node.

root@mongo1:# mongod -f /etc/mongodb1.conf &

3) Log into mongo database and initialize replica.

root@mongo1:~# mongo --port 17017
MongoDB shell version: 2.0.6
connecting to: 127.0.0.1:17017/test
> rs.config()
null
> rs.status()
{
 "startupStatus" : 3,
 "info" : "run rs.initiate(...) if not yet done for the set",
 "errmsg" : "can't get local.system.replset config from self or any seed (EMPTYCONFIG)",
 "ok" : 0
}
> rs.initiate();
{
 "info2" : "no configuration explicitly specified -- making one",
 "me" : "mongo1:17017",
 "info" : "Config now saved locally.  Should come online in about a minute.",
 "ok" : 1
}

> rs.status()
{
 "set" : "replication_test",
 "date" : ISODate("2015-07-27T14:30:34Z"),
 "myState" : 2,
 "members" : [
  {
   "_id" : 0,
   "name" : "mongo1:17017",
   "health" : 1,
   "state" : 2,
   "stateStr" : "SECONDARY",
   "optime" : {
    "t" : 1438007418000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:30:18Z"),
   "self" : true
  }
 ],
 "ok" : 1
}
SECONDARY> rs.status()
{
 "set" : "replication_test",
 "date" : ISODate("2015-07-27T14:31:01Z"),
 "myState" : 1,
 "members" : [
  {
   "_id" : 0,
   "name" : "mongo1:17017",
   "health" : 1,
   "state" : 1,
   "stateStr" : "PRIMARY",
   "optime" : {
    "t" : 1438007418000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:30:18Z"),
   "self" : true
  }
 ],
 "ok" : 1
}
PRIMARY> rs.config()
{
 "_id" : "replication_test",
 "version" : 1,
 "members" : [
  {
   "_id" : 0,
   "host" : "mongo1:17017"
  }
 ]
}

4) Log into second node and start mongod:

root@mongo2:~# mongod -f /etc/mongodb2.conf &

cat /var/log/mongodb/mongodb2.log

Mon Jul 27 14:32:55 [initandlisten] MongoDB starting : pid=2650 port=27017 dbpath=/var/lib/mongodb2 64-bit host=mongo
Mon Jul 27 14:32:55 [initandlisten] 
Mon Jul 27 14:32:55 [initandlisten] ** WARNING: You are running on a NUMA machine.
Mon Jul 27 14:32:55 [initandlisten] **          We suggest launching mongod like this to avoid performance problems:
Mon Jul 27 14:32:55 [initandlisten] **              numactl --interleave=all mongod [other options]
Mon Jul 27 14:32:55 [initandlisten] 
Mon Jul 27 14:32:55 [initandlisten] db version v2.0.6, pdfile version 4.5
Mon Jul 27 14:32:55 [initandlisten] git version: nogitversion
Mon Jul 27 14:32:55 [initandlisten] build info: Linux z6 3.8-trunk-amd64 #1 SMP Debian 3.8.3-1~experimental.1 x86_64 BOOST_LIB_VERSION=1_49
Mon Jul 27 14:32:55 [initandlisten] options: { bind_ip: "mongo", config: "/etc/mongodb2.conf", dbpath: "/var/lib/mongodb2", journal: "true", logappend: "true", logpath: "/var/log/mongodb/mongodb2.log", port: 27017, replSet: "replication_test" }
Mon Jul 27 14:32:55 [initandlisten] journal dir=/var/lib/mongodb2/journal
Mon Jul 27 14:32:55 [initandlisten] recover : no journal files present, no recovery needed
Mon Jul 27 14:32:55 [initandlisten] waiting for connections on port 27017
Mon Jul 27 14:32:55 [websvr] admin web console waiting for connections on port 28017
Mon Jul 27 14:32:55 [initandlisten] connection accepted from 10.0.0.3:42241 #1
Mon Jul 27 14:32:55 [rsStart] replSet can't get local.system.replset config from self or any seed (EMPTYCONFIG)
Mon Jul 27 14:32:55 [rsStart] replSet info you may need to run replSetInitiate -- rs.initiate() in the shell -- if that is not already done

5) On first node ( PRIMARY) add secondary node:

PRIMARY> rs.add("mongo2:27017")
{ "ok" : 1 }
PRIMARY> rs.status()
{
 "set" : "replication_test",
 "date" : ISODate("2015-07-27T14:35:31Z"),
 "myState" : 1,
 "members" : [
  {
   "_id" : 0,
   "name" : "mongo1:17017",
   "health" : 1,
   "state" : 1,
   "stateStr" : "PRIMARY",
   "optime" : {
    "t" : 1438007724000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:35:24Z"),
   "self" : true
  },
  {
   "_id" : 1,
   "name" : "mongo2:27017",
   "health" : 1,
   "state" : 3,
   "stateStr" : "RECOVERING",
   "uptime" : 7,
   "optime" : {
    "t" : 0,
    "i" : 0
   },
   "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:35:30Z"),
   "pingMs" : 21545440,
   "errmsg" : "initial sync need a member to be primary or secondary to do our initial sync"
  }
 ],
 "ok" : 1
}

PRIMARY> rs.status()
{
 "set" : "replication_test",
 "date" : ISODate("2015-07-27T14:36:32Z"),
 "myState" : 1,
 "members" : [
  {
   "_id" : 0,
   "name" : "mongo1:17017",
   "health" : 1,
   "state" : 1,
   "stateStr" : "PRIMARY",
   "optime" : {
    "t" : 1438007724000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:35:24Z"),
   "self" : true
  },
  {
   "_id" : 1,
   "name" : "mongo2:27017",
   "health" : 1,
   "state" : 2,
   "stateStr" : "SECONDARY",
   "uptime" : 68,
   "optime" : {
    "t" : 1438007724000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:35:24Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:36:30Z"),
   "pingMs" : 26670
  }
 ],
 "ok" : 1
}

6) Run arbiter on the third node and add it to exist infrastructure using PRIMARY node:

root@mongo3:~# mongod -f /etc/mongodb3.conf &

root@mongo1:~# mongo --port 17017
PRIMARY> rs.addArb("mongo3:37017");
{ "ok" : 1 }
PRIMARY> rs.status();
{
 "set" : "replication_test",
 "date" : ISODate("2015-07-27T14:39:09Z"),
 "myState" : 1,
 "members" : [
  {
   "_id" : 0,
   "name" : "mongo1:17017",
   "health" : 1,
   "state" : 1,
   "stateStr" : "PRIMARY",
   "optime" : {
    "t" : 1438007937000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:38:57Z"),
   "self" : true
  },
  {
   "_id" : 1,
   "name" : "mongo2:27017",
   "health" : 1,
   "state" : 2,
   "stateStr" : "SECONDARY",
   "uptime" : 225,
   "optime" : {
    "t" : 1438007937000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:38:57Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:39:08Z"),
   "pingMs" : 0
  },
  {
   "_id" : 2,
   "name" : "mongo3:37017",
   "health" : 1,
   "state" : 5,
   "stateStr" : "STARTUP2",
   "uptime" : 12,
   "optime" : {
    "t" : 0,
    "i" : 0
   },
   "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:39:07Z"),
   "pingMs" : 0
  }
 ],
 "ok" : 1
}
PRIMARY> rs.status();
{
 "set" : "replication_test",
 "date" : ISODate("2015-07-27T14:39:14Z"),
 "myState" : 1,
 "members" : [
  {
   "_id" : 0,
   "name" : "mongo1:17017",
   "health" : 1,
   "state" : 1,
   "stateStr" : "PRIMARY",
   "optime" : {
    "t" : 1438007937000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:38:57Z"),
   "self" : true
  },
  {
   "_id" : 1,
   "name" : "mongo2:27017",
   "health" : 1,
   "state" : 2,
   "stateStr" : "SECONDARY",
   "uptime" : 230,
   "optime" : {
    "t" : 1438007937000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:38:57Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:39:12Z"),
   "pingMs" : 0
  },
  {
   "_id" : 2,
   "name" : "mongo3:37017",
   "health" : 1,
   "state" : 7,
   "stateStr" : "ARBITER",
   "uptime" : 17,
   "optime" : {
    "t" : 0,
    "i" : 0
   },
   "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:39:13Z"),
   "pingMs" : 0
  }
 ],
 "ok" : 1
}

PRIMARY> rs.config()
{
 "_id" : "replication_test",
 "version" : 3,
 "members" : [
  {
   "_id" : 0,
   "host" : "mongo1:17017"
  },
  {
   "_id" : 1,
   "host" : "mongo2:27017"
  },
  {
   "_id" : 2,
   "host" : "mongo3:37017",
   "arbiterOnly" : true
  }
 ]
}

Let's see what happen if I kill PRIMARY:

root@mongo2:~# mongo --port 27017
MongoDB shell version: 2.0.6
connecting to: 127.0.0.1:27017/test
PRIMARY> rs.status()
{
 "set" : "replication_test",
 "date" : ISODate("2015-07-27T14:41:33Z"),
 "myState" : 1,
 "syncingTo" : "mongo1:17017",
 "members" : [
  {
   "_id" : 0,
   "name" : "mongo1:17017",
   "health" : 0,
   "state" : 8,
   "stateStr" : "(not reachable/healthy)",
   "uptime" : 0,
   "optime" : {
    "t" : 1438007937000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:38:57Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:41:13Z"),
   "pingMs" : 0,
   "errmsg" : "socket exception"
  },
  {
   "_id" : 1,
   "name" : "mongo2:27017",
   "health" : 1,
   "state" : 1,
   "stateStr" : "PRIMARY",
   "optime" : {
    "t" : 1438007937000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:38:57Z"),
   "self" : true
  },
  {
   "_id" : 2,
   "name" : "mongo3:37017",
   "health" : 1,
   "state" : 7,
   "stateStr" : "ARBITER",
   "uptime" : 154,
   "optime" : {
    "t" : 0,
    "i" : 0
   },
   "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:41:33Z"),
   "pingMs" : 0
  }
 ],
 "ok" : 1
}

As we assumed SECONDARY becomes PRIMARY.

And if I start old PRIMARY again ?

root@mongo1:~# mongod -f /etc/mongodb1.conf &

Yes, it'll be SECONDARY.

root@mongo2:~# mongo --port 27017
MongoDB shell version: 2.0.6
connecting to: 127.0.0.1:27017/test
PRIMARY> rs.status();
{
 "set" : "replication_test",
 "date" : ISODate("2015-07-27T14:42:53Z"),
 "myState" : 1,
 "syncingTo" : "mongo1:17017",
 "members" : [
  {
   "_id" : 0,
   "name" : "mongo1:17017",
   "health" : 1,
   "state" : 2,
   "stateStr" : "SECONDARY",
   "uptime" : 6,
   "optime" : {
    "t" : 1438007937000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:38:57Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:42:53Z"),
   "pingMs" : 0
  },
  {
   "_id" : 1,
   "name" : "mongo2:27017",
   "health" : 1,
   "state" : 1,
   "stateStr" : "PRIMARY",
   "optime" : {
    "t" : 1438007937000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:38:57Z"),
   "self" : true
  },
  {
   "_id" : 2,
   "name" : "mongo3:37017",
   "health" : 1,
   "state" : 7,
   "stateStr" : "ARBITER",
   "uptime" : 234,
   "optime" : {
    "t" : 0,
    "i" : 0
   },
   "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:42:53Z"),
   "pingMs" : 0
  }
 ],
 "ok" : 1
}

Of course I can turn my SECONDARY into PRIMARY using "rs" command:

root@mongo2:~# mongo --port 27017
MongoDB shell version: 2.0.6
connecting to: 127.0.0.1:27017/test
PRIMARY> rs.stepDown(120)
Mon Jul 27 14:46:12 DBClientCursor::init call() failed
Mon Jul 27 14:46:12 query failed : admin.$cmd { replSetStepDown: 120.0 } to: 127.0.0.1:27017
Mon Jul 27 14:46:12 Error: error doing query: failed shell/collection.js:151
Mon Jul 27 14:46:12 trying reconnect to 127.0.0.1:27017
Mon Jul 27 14:46:12 reconnect 127.0.0.1:27017 ok
SECONDARY> rs.status()
{
 "set" : "replication_test",
 "date" : ISODate("2015-07-27T14:46:30Z"),
 "myState" : 2,
 "syncingTo" : "mongo1:17017",
 "members" : [
  {
   "_id" : 0,
   "name" : "mongo1:17017",
   "health" : 1,
   "state" : 1,
   "stateStr" : "PRIMARY",
   "uptime" : 15,
   "optime" : {
    "t" : 1438007937000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:38:57Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:46:29Z"),
   "pingMs" : 0
  },
  {
   "_id" : 1,
   "name" : "mongo2:27017",
   "health" : 1,
   "state" : 2,
   "stateStr" : "SECONDARY",
   "optime" : {
    "t" : 1438007937000,
    "i" : 1
   },
   "optimeDate" : ISODate("2015-07-27T14:38:57Z"),
   "self" : true
  },
  {
   "_id" : 2,
   "name" : "mongo3:37017",
   "health" : 1,
   "state" : 7,
   "stateStr" : "ARBITER",
   "uptime" : 15,
   "optime" : {
    "t" : 0,
    "i" : 0
   },
   "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
   "lastHeartbeat" : ISODate("2015-07-27T14:46:29Z"),
   "pingMs" : 0
  }
 ],
 "ok" : 1
}

Share my minds

wtorek, 28 lipca 2015

MongoDB replication with arbiter

How to set up

Szukaj na tym blogu