Details
-
Bug
-
Status: Done
-
Medium
-
Resolution: Fixed
-
1.13.0
-
None
-
None
-
Yes
-
Yes
-
Yes
Description
As reported on slack, if you run replica set with 1.12 and expose set to LoadBalancer for replica nodes and you switch to ClusterIP later the replica will be reconfigured with these new IPs.
With 1.13 it's not the case, but the cluster is left in initializing state.
I think it's possible that in 1.12 it worked because we had force option enabled on replica reconfigure, but we have removed it because of other problems this creates.
Here's how it looks:
1.13 before:
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE my-cluster-name-rs0 ClusterIP None <none> 27017/TCP 119s my-cluster-name-rs0-0 LoadBalancer 10.58.120.212 34.27.80.226 27017:31110/TCP 116s my-cluster-name-rs0-1 LoadBalancer 10.58.115.211 34.135.217.111 27017:30439/TCP 72s my-cluster-name-rs0-2 LoadBalancer 10.58.113.110 34.66.45.46 27017:31794/TCP 36s rs0:PRIMARY> rs.config() { "_id" : "rs0", "version" : 8, "term" : 1, "members" : [ { "_id" : 0, "host" : "34.27.80.226:27017", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 2, "tags" : { "podName" : "my-cluster-name-rs0-0", "serviceName" : "my-cluster-name" }, "secondaryDelaySecs" : NumberLong(0), "votes" : 1 }, { "_id" : 1, "host" : "34.135.217.111:27017", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 2, "tags" : { "serviceName" : "my-cluster-name", "podName" : "my-cluster-name-rs0-1" }, "secondaryDelaySecs" : NumberLong(0), "votes" : 1 }, { "_id" : 2, "host" : "34.66.45.46:27017", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 2, "tags" : { "podName" : "my-cluster-name-rs0-2", "serviceName" : "my-cluster-name" }, "secondaryDelaySecs" : NumberLong(0), "votes" : 1 } ], "protocolVersion" : NumberLong(1), "writeConcernMajorityJournalDefault" : true, "settings" : { "chainingAllowed" : true, "heartbeatIntervalMillis" : 2000, "heartbeatTimeoutSecs" : 10, "electionTimeoutMillis" : 10000, "catchUpTimeoutMillis" : -1, "catchUpTakeoverDelayMillis" : 30000, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 }, "replicaSetId" : ObjectId("63c7e02e0f7487651a4cd4d8") } } rs0:PRIMARY> rs.status() { "set" : "rs0", "date" : ISODate("2023-01-18T12:05:21.338Z"), "myState" : 1, "term" : NumberLong(1), "syncSourceHost" : "", "syncSourceId" : -1, "heartbeatIntervalMillis" : NumberLong(2000), "majorityVoteCount" : 2, "writeMajorityCount" : 2, "votingMembersCount" : 3, "writableVotingMembersCount" : 3, "optimes" : { "lastCommittedOpTime" : { "ts" : Timestamp(1674043519, 4), "t" : NumberLong(1) }, "lastCommittedWallTime" : ISODate("2023-01-18T12:05:19.163Z"), "readConcernMajorityOpTime" : { "ts" : Timestamp(1674043519, 4), "t" : NumberLong(1) }, "appliedOpTime" : { "ts" : Timestamp(1674043519, 4), "t" : NumberLong(1) }, "durableOpTime" : { "ts" : Timestamp(1674043519, 4), "t" : NumberLong(1) }, "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"), "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z") }, "lastStableRecoveryTimestamp" : Timestamp(1674043494, 2), "electionCandidateMetrics" : { "lastElectionReason" : "electionTimeout", "lastElectionDate" : ISODate("2023-01-18T12:03:58.335Z"), "electionTerm" : NumberLong(1), "lastCommittedOpTimeAtElection" : { "ts" : Timestamp(1674043438, 1), "t" : NumberLong(-1) }, "lastSeenOpTimeAtElection" : { "ts" : Timestamp(1674043438, 1), "t" : NumberLong(-1) }, "numVotesNeeded" : 1, "priorityAtElection" : 1, "electionTimeoutMillis" : NumberLong(10000), "newTermStartDate" : ISODate("2023-01-18T12:03:58.414Z"), "wMajorityWriteAvailabilityDate" : ISODate("2023-01-18T12:03:58.433Z") }, "members" : [ { "_id" : 0, "name" : "34.27.80.226:27017", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 166, "optime" : { "ts" : Timestamp(1674043519, 4), "t" : NumberLong(1) }, "optimeDate" : ISODate("2023-01-18T12:05:19Z"), "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"), "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z"), "syncSourceHost" : "", "syncSourceId" : -1, "infoMessage" : "Could not find member to sync from", "electionTime" : Timestamp(1674043438, 2), "electionDate" : ISODate("2023-01-18T12:03:58Z"), "configVersion" : 8, "configTerm" : 1, "self" : true, "lastHeartbeatMessage" : "" }, { "_id" : 1, "name" : "34.135.217.111:27017", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 38, "optime" : { "ts" : Timestamp(1674043519, 4), "t" : NumberLong(1) }, "optimeDurable" : { "ts" : Timestamp(1674043519, 4), "t" : NumberLong(1) }, "optimeDate" : ISODate("2023-01-18T12:05:19Z"), "optimeDurableDate" : ISODate("2023-01-18T12:05:19Z"), "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"), "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z"), "lastHeartbeat" : ISODate("2023-01-18T12:05:20.649Z"), "lastHeartbeatRecv" : ISODate("2023-01-18T12:05:20.677Z"), "pingMs" : NumberLong(0), "lastHeartbeatMessage" : "", "syncSourceHost" : "34.27.80.226:27017", "syncSourceId" : 0, "infoMessage" : "", "configVersion" : 8, "configTerm" : 1 }, { "_id" : 2, "name" : "34.66.45.46:27017", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 44, "optime" : { "ts" : Timestamp(1674043519, 4), "t" : NumberLong(1) }, "optimeDurable" : { "ts" : Timestamp(1674043519, 4), "t" : NumberLong(1) }, "optimeDate" : ISODate("2023-01-18T12:05:19Z"), "optimeDurableDate" : ISODate("2023-01-18T12:05:19Z"), "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"), "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z"), "lastHeartbeat" : ISODate("2023-01-18T12:05:20.649Z"), "lastHeartbeatRecv" : ISODate("2023-01-18T12:05:21.221Z"), "pingMs" : NumberLong(0), "lastHeartbeatMessage" : "", "syncSourceHost" : "34.135.217.111:27017", "syncSourceId" : 1, "infoMessage" : "", "configVersion" : 8, "configTerm" : 1 } ], "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1674043519, 4), "signature" : { "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="), "keyId" : NumberLong(0) } }, "operationTime" : Timestamp(1674043519, 4) }
1.13 after:
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE my-cluster-name-rs0 ClusterIP None <none> 27017/TCP 8m25s my-cluster-name-rs0-0 ClusterIP 10.58.120.212 <none> 27017/TCP 8m22s my-cluster-name-rs0-1 ClusterIP 10.58.115.211 <none> 27017/TCP 7m38s my-cluster-name-rs0-2 ClusterIP 10.58.113.110 <none> 27017/TCP 7m2s NAME ENDPOINT STATUS AGE my-cluster-name my-cluster-name-rs0.test.svc.cluster.local initializing 9m32s $ psmdb-client.sh mongodb_uri: mongodb://backup:back[email protected]/admin?ssl=false&authSource=admin&replicaSet=rs0 If you don't see a command prompt, try pressing enter. {"t":{"$date":"2023-01-18T10:02:36.741Z"},"s":"I", "c":"NETWORK", "id":4333208, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM host selection timeout","attr":{"replicaSet":"rs0","error":"FailedToSatisfyReadPreference: Could not find host matching read preference { mode: \"nearest\" } for set rs0"}} Error: connect failed to replica set rs0/my-cluster-name-rs0-0.test.svc.cluster.local:27017 : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 pod "percona-client-4946" deleted pod test/percona-client-4946 terminated (Error)
1.13 operator log: operator.log
1.12 before:
# # 1.12 before # NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE my-cluster-name-rs0 ClusterIP None <none> 27017/TCP 2m34s my-cluster-name-rs0-0 LoadBalancer 10.58.113.255 34.136.125.249 27017:31978/TCP 2m31s my-cluster-name-rs0-1 LoadBalancer 10.58.124.240 35.194.50.109 27017:30905/TCP 105s my-cluster-name-rs0-2 LoadBalancer 10.58.124.61 34.135.119.222 27017:31863/TCP 71s rs0:PRIMARY> rs.status() { "set" : "rs0", "date" : ISODate("2023-01-18T11:52:08.778Z"), "myState" : 1, "term" : NumberLong(1), "syncSourceHost" : "", "syncSourceId" : -1, "heartbeatIntervalMillis" : NumberLong(2000), "majorityVoteCount" : 2, "writeMajorityCount" : 2, "votingMembersCount" : 3, "writableVotingMembersCount" : 3, "optimes" : { "lastCommittedOpTime" : { "ts" : Timestamp(1674042728, 1), "t" : NumberLong(1) }, "lastCommittedWallTime" : ISODate("2023-01-18T11:52:08.708Z"), "readConcernMajorityOpTime" : { "ts" : Timestamp(1674042728, 1), "t" : NumberLong(1) }, "appliedOpTime" : { "ts" : Timestamp(1674042728, 1), "t" : NumberLong(1) }, "durableOpTime" : { "ts" : Timestamp(1674042728, 1), "t" : NumberLong(1) }, "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"), "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z") }, "lastStableRecoveryTimestamp" : Timestamp(1674042679, 6), "electionCandidateMetrics" : { "lastElectionReason" : "electionTimeout", "lastElectionDate" : ISODate("2023-01-18T11:50:23.336Z"), "electionTerm" : NumberLong(1), "lastCommittedOpTimeAtElection" : { "ts" : Timestamp(1674042623, 1), "t" : NumberLong(-1) }, "lastSeenOpTimeAtElection" : { "ts" : Timestamp(1674042623, 1), "t" : NumberLong(-1) }, "numVotesNeeded" : 1, "priorityAtElection" : 1, "electionTimeoutMillis" : NumberLong(10000), "newTermStartDate" : ISODate("2023-01-18T11:50:23.425Z"), "wMajorityWriteAvailabilityDate" : ISODate("2023-01-18T11:50:23.441Z") }, "members" : [ { "_id" : 0, "name" : "34.136.125.249:27017", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 175, "optime" : { "ts" : Timestamp(1674042728, 1), "t" : NumberLong(1) }, "optimeDate" : ISODate("2023-01-18T11:52:08Z"), "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"), "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z"), "syncSourceHost" : "", "syncSourceId" : -1, "infoMessage" : "Could not find member to sync from", "electionTime" : Timestamp(1674042623, 2), "electionDate" : ISODate("2023-01-18T11:50:23Z"), "configVersion" : 53733, "configTerm" : -1, "self" : true, "lastHeartbeatMessage" : "" }, { "_id" : 1, "name" : "35.194.50.109:27017", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 79, "optime" : { "ts" : Timestamp(1674042724, 6), "t" : NumberLong(1) }, "optimeDurable" : { "ts" : Timestamp(1674042724, 6), "t" : NumberLong(1) }, "optimeDate" : ISODate("2023-01-18T11:52:04Z"), "optimeDurableDate" : ISODate("2023-01-18T11:52:04Z"), "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"), "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z"), "lastHeartbeat" : ISODate("2023-01-18T11:52:07.740Z"), "lastHeartbeatRecv" : ISODate("2023-01-18T11:52:07.999Z"), "pingMs" : NumberLong(0), "lastHeartbeatMessage" : "", "syncSourceHost" : "34.136.125.249:27017", "syncSourceId" : 0, "infoMessage" : "", "configVersion" : 53733, "configTerm" : -1 }, { "_id" : 2, "name" : "34.135.119.222:27017", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 79, "optime" : { "ts" : Timestamp(1674042724, 6), "t" : NumberLong(1) }, "optimeDurable" : { "ts" : Timestamp(1674042724, 6), "t" : NumberLong(1) }, "optimeDate" : ISODate("2023-01-18T11:52:04Z"), "optimeDurableDate" : ISODate("2023-01-18T11:52:04Z"), "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"), "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z"), "lastHeartbeat" : ISODate("2023-01-18T11:52:07.740Z"), "lastHeartbeatRecv" : ISODate("2023-01-18T11:52:06.895Z"), "pingMs" : NumberLong(0), "lastHeartbeatMessage" : "", "syncSourceHost" : "35.194.50.109:27017", "syncSourceId" : 1, "infoMessage" : "", "configVersion" : 53733, "configTerm" : -1 } ], "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1674042728, 1), "signature" : { "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="), "keyId" : NumberLong(0) } }, "operationTime" : Timestamp(1674042728, 1) } rs0:PRIMARY> rs.config() { "_id" : "rs0", "version" : 53733, "members" : [ { "_id" : 0, "host" : "34.136.125.249:27017", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 2, "tags" : { "podName" : "my-cluster-name-rs0-0", "serviceName" : "my-cluster-name" }, "secondaryDelaySecs" : NumberLong(0), "votes" : 1 }, { "_id" : 1, "host" : "35.194.50.109:27017", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 2, "tags" : { "podName" : "my-cluster-name-rs0-1", "serviceName" : "my-cluster-name" }, "secondaryDelaySecs" : NumberLong(0), "votes" : 1 }, { "_id" : 2, "host" : "34.135.119.222:27017", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 2, "tags" : { "podName" : "my-cluster-name-rs0-2", "serviceName" : "my-cluster-name" }, "secondaryDelaySecs" : NumberLong(0), "votes" : 1 } ], "protocolVersion" : NumberLong(1), "writeConcernMajorityJournalDefault" : true, "settings" : { "chainingAllowed" : true, "heartbeatIntervalMillis" : 2000, "heartbeatTimeoutSecs" : 10, "electionTimeoutMillis" : 10000, "catchUpTimeoutMillis" : -1, "catchUpTakeoverDelayMillis" : 30000, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 }, "replicaSetId" : ObjectId("63c7dcff2a02a8a4a5e2375d") } }
1.12 after:
# # 1.12 after # NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE my-cluster-name-rs0 ClusterIP None <none> 27017/TCP 5m13s my-cluster-name-rs0-0 ClusterIP 10.58.113.255 <none> 27017/TCP 5m10s my-cluster-name-rs0-1 ClusterIP 10.58.124.240 <none> 27017/TCP 4m24s my-cluster-name-rs0-2 ClusterIP 10.58.124.61 <none> 27017/TCP 3m50s NAME ENDPOINT STATUS AGE my-cluster-name my-cluster-name-rs0.test.svc.cluster.local ready 5m49s rs0:PRIMARY> rs.status() { "set" : "rs0", "date" : ISODate("2023-01-18T11:55:16.135Z"), "myState" : 1, "term" : NumberLong(1), "syncSourceHost" : "", "syncSourceId" : -1, "heartbeatIntervalMillis" : NumberLong(2000), "majorityVoteCount" : 2, "writeMajorityCount" : 2, "votingMembersCount" : 3, "writableVotingMembersCount" : 3, "optimes" : { "lastCommittedOpTime" : { "ts" : Timestamp(1674042915, 1), "t" : NumberLong(1) }, "lastCommittedWallTime" : ISODate("2023-01-18T11:55:15.570Z"), "readConcernMajorityOpTime" : { "ts" : Timestamp(1674042915, 1), "t" : NumberLong(1) }, "appliedOpTime" : { "ts" : Timestamp(1674042915, 1), "t" : NumberLong(1) }, "durableOpTime" : { "ts" : Timestamp(1674042915, 1), "t" : NumberLong(1) }, "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"), "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z") }, "lastStableRecoveryTimestamp" : Timestamp(1674042857, 1), "electionCandidateMetrics" : { "lastElectionReason" : "electionTimeout", "lastElectionDate" : ISODate("2023-01-18T11:50:23.336Z"), "electionTerm" : NumberLong(1), "lastCommittedOpTimeAtElection" : { "ts" : Timestamp(1674042623, 1), "t" : NumberLong(-1) }, "lastSeenOpTimeAtElection" : { "ts" : Timestamp(1674042623, 1), "t" : NumberLong(-1) }, "numVotesNeeded" : 1, "priorityAtElection" : 1, "electionTimeoutMillis" : NumberLong(10000), "newTermStartDate" : ISODate("2023-01-18T11:50:23.425Z"), "wMajorityWriteAvailabilityDate" : ISODate("2023-01-18T11:50:23.441Z") }, "members" : [ { "_id" : 0, "name" : "10.58.113.255:27017", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 363, "optime" : { "ts" : Timestamp(1674042915, 1), "t" : NumberLong(1) }, "optimeDate" : ISODate("2023-01-18T11:55:15Z"), "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"), "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z"), "syncSourceHost" : "", "syncSourceId" : -1, "infoMessage" : "", "electionTime" : Timestamp(1674042623, 2), "electionDate" : ISODate("2023-01-18T11:50:23Z"), "configVersion" : 202648, "configTerm" : -1, "self" : true, "lastHeartbeatMessage" : "" }, { "_id" : 1, "name" : "10.58.124.240:27017", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 58, "optime" : { "ts" : Timestamp(1674042915, 1), "t" : NumberLong(1) }, "optimeDurable" : { "ts" : Timestamp(1674042915, 1), "t" : NumberLong(1) }, "optimeDate" : ISODate("2023-01-18T11:55:15Z"), "optimeDurableDate" : ISODate("2023-01-18T11:55:15Z"), "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"), "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z"), "lastHeartbeat" : ISODate("2023-01-18T11:55:15.629Z"), "lastHeartbeatRecv" : ISODate("2023-01-18T11:55:14.451Z"), "pingMs" : NumberLong(0), "lastHeartbeatMessage" : "", "syncSourceHost" : "10.58.113.255:27017", "syncSourceId" : 0, "infoMessage" : "", "configVersion" : 202648, "configTerm" : -1 }, { "_id" : 2, "name" : "10.58.124.61:27017", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 58, "optime" : { "ts" : Timestamp(1674042915, 1), "t" : NumberLong(1) }, "optimeDurable" : { "ts" : Timestamp(1674042915, 1), "t" : NumberLong(1) }, "optimeDate" : ISODate("2023-01-18T11:55:15Z"), "optimeDurableDate" : ISODate("2023-01-18T11:55:15Z"), "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"), "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z"), "lastHeartbeat" : ISODate("2023-01-18T11:55:15.629Z"), "lastHeartbeatRecv" : ISODate("2023-01-18T11:55:14.451Z"), "pingMs" : NumberLong(0), "lastHeartbeatMessage" : "", "syncSourceHost" : "10.58.113.255:27017", "syncSourceId" : 0, "infoMessage" : "", "configVersion" : 202648, "configTerm" : -1 } ], "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1674042915, 1), "signature" : { "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="), "keyId" : NumberLong(0) } }, "operationTime" : Timestamp(1674042915, 1) } rs0:PRIMARY> rs.config() { "_id" : "rs0", "version" : 202648, "members" : [ { "_id" : 0, "host" : "10.58.113.255:27017", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 2, "tags" : { "serviceName" : "my-cluster-name", "podName" : "my-cluster-name-rs0-0" }, "secondaryDelaySecs" : NumberLong(0), "votes" : 1 }, { "_id" : 1, "host" : "10.58.124.240:27017", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 2, "tags" : { "podName" : "my-cluster-name-rs0-1", "serviceName" : "my-cluster-name" }, "secondaryDelaySecs" : NumberLong(0), "votes" : 1 }, { "_id" : 2, "host" : "10.58.124.61:27017", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 2, "tags" : { "serviceName" : "my-cluster-name", "podName" : "my-cluster-name-rs0-2" }, "secondaryDelaySecs" : NumberLong(0), "votes" : 1 } ], "protocolVersion" : NumberLong(1), "writeConcernMajorityJournalDefault" : true, "settings" : { "chainingAllowed" : true, "heartbeatIntervalMillis" : 2000, "heartbeatTimeoutSecs" : 10, "electionTimeoutMillis" : 10000, "catchUpTimeoutMillis" : -1, "catchUpTakeoverDelayMillis" : 30000, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 }, "replicaSetId" : ObjectId("63c7dcff2a02a8a4a5e2375d") } }