Uploaded image for project: 'Percona Operator for MongoDB'
  1. Percona Operator for MongoDB
  2. K8SPSMDB-841

Cluster unready after switching from expose LoadBalancer to ClusterIP

Details

    • Bug
    • Status: Done
    • Medium
    • Resolution: Fixed
    • 1.13.0
    • 1.14.0
    • None
    • None
    • Yes
    • Yes
    • Yes

    Description

      As reported on slack, if you run replica set with 1.12 and expose set to LoadBalancer for replica nodes and you switch to ClusterIP later the replica will be reconfigured with these new IPs.

      With 1.13 it's not the case, but the cluster is left in initializing state.
      I think it's possible that in 1.12 it worked because we had force option enabled on replica reconfigure, but we have removed it because of other problems this creates.

      Here's how it looks:
      1.13 before:

      NAME                    TYPE           CLUSTER-IP      EXTERNAL-IP      PORT(S)           AGE
      my-cluster-name-rs0     ClusterIP      None            <none>           27017/TCP         119s
      my-cluster-name-rs0-0   LoadBalancer   10.58.120.212   34.27.80.226     27017:31110/TCP   116s
      my-cluster-name-rs0-1   LoadBalancer   10.58.115.211   34.135.217.111   27017:30439/TCP   72s
      my-cluster-name-rs0-2   LoadBalancer   10.58.113.110   34.66.45.46      27017:31794/TCP   36s
      
      rs0:PRIMARY> rs.config()
      {
              "_id" : "rs0",
              "version" : 8,
              "term" : 1,
              "members" : [
                      {
                              "_id" : 0,
                              "host" : "34.27.80.226:27017",
                              "arbiterOnly" : false,
                              "buildIndexes" : true,
                              "hidden" : false,
                              "priority" : 2,
                              "tags" : {
                                      "podName" : "my-cluster-name-rs0-0",
                                      "serviceName" : "my-cluster-name"
                              },
                              "secondaryDelaySecs" : NumberLong(0),
                              "votes" : 1
                      },
                      {
                              "_id" : 1,
                              "host" : "34.135.217.111:27017",
                              "arbiterOnly" : false,
                              "buildIndexes" : true,
                              "hidden" : false,
                              "priority" : 2,
                              "tags" : {
                                      "serviceName" : "my-cluster-name",
                                      "podName" : "my-cluster-name-rs0-1"
                              },
                              "secondaryDelaySecs" : NumberLong(0),
                              "votes" : 1
                      },
                      {
                              "_id" : 2,
                              "host" : "34.66.45.46:27017",
                              "arbiterOnly" : false,
                              "buildIndexes" : true,
                              "hidden" : false,
                              "priority" : 2,
                              "tags" : {
                                      "podName" : "my-cluster-name-rs0-2",
                                      "serviceName" : "my-cluster-name"
                              },
                              "secondaryDelaySecs" : NumberLong(0),
                              "votes" : 1
                      }
              ],
              "protocolVersion" : NumberLong(1),
              "writeConcernMajorityJournalDefault" : true,
              "settings" : {
                      "chainingAllowed" : true,
                      "heartbeatIntervalMillis" : 2000,
                      "heartbeatTimeoutSecs" : 10,
                      "electionTimeoutMillis" : 10000,
                      "catchUpTimeoutMillis" : -1,
                      "catchUpTakeoverDelayMillis" : 30000,
                      "getLastErrorModes" : {
      
                      },
                      "getLastErrorDefaults" : {
                              "w" : 1,
                              "wtimeout" : 0
                      },
                      "replicaSetId" : ObjectId("63c7e02e0f7487651a4cd4d8")
              }
      }
      
      rs0:PRIMARY> rs.status()
      {
              "set" : "rs0",
              "date" : ISODate("2023-01-18T12:05:21.338Z"),
              "myState" : 1,
              "term" : NumberLong(1),
              "syncSourceHost" : "",
              "syncSourceId" : -1,
              "heartbeatIntervalMillis" : NumberLong(2000),
              "majorityVoteCount" : 2,
              "writeMajorityCount" : 2,
              "votingMembersCount" : 3,
              "writableVotingMembersCount" : 3,
              "optimes" : {
                      "lastCommittedOpTime" : {
                              "ts" : Timestamp(1674043519, 4),
                              "t" : NumberLong(1)
                      },
                      "lastCommittedWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                      "readConcernMajorityOpTime" : {
                              "ts" : Timestamp(1674043519, 4),
                              "t" : NumberLong(1)
                      },
                      "appliedOpTime" : {
                              "ts" : Timestamp(1674043519, 4),
                              "t" : NumberLong(1)
                      },
                      "durableOpTime" : {
                              "ts" : Timestamp(1674043519, 4),
                              "t" : NumberLong(1)
                      },
                      "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                      "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z")
              },
              "lastStableRecoveryTimestamp" : Timestamp(1674043494, 2),
              "electionCandidateMetrics" : {
                      "lastElectionReason" : "electionTimeout",
                      "lastElectionDate" : ISODate("2023-01-18T12:03:58.335Z"),
                      "electionTerm" : NumberLong(1),
                      "lastCommittedOpTimeAtElection" : {
                              "ts" : Timestamp(1674043438, 1),
                              "t" : NumberLong(-1)
                      },
                      "lastSeenOpTimeAtElection" : {
                              "ts" : Timestamp(1674043438, 1),
                              "t" : NumberLong(-1)
                      },
                      "numVotesNeeded" : 1,
                      "priorityAtElection" : 1,
                      "electionTimeoutMillis" : NumberLong(10000),
                      "newTermStartDate" : ISODate("2023-01-18T12:03:58.414Z"),
                      "wMajorityWriteAvailabilityDate" : ISODate("2023-01-18T12:03:58.433Z")
              },
              "members" : [
                      {
                              "_id" : 0,
                              "name" : "34.27.80.226:27017",
                              "health" : 1,
                              "state" : 1,
                              "stateStr" : "PRIMARY",
                              "uptime" : 166,
                              "optime" : {
                                      "ts" : Timestamp(1674043519, 4),
                                      "t" : NumberLong(1)
                              },
                              "optimeDate" : ISODate("2023-01-18T12:05:19Z"),
                              "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                              "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                              "syncSourceHost" : "",
                              "syncSourceId" : -1,
                              "infoMessage" : "Could not find member to sync from",
                              "electionTime" : Timestamp(1674043438, 2),
                              "electionDate" : ISODate("2023-01-18T12:03:58Z"),
                              "configVersion" : 8,
                              "configTerm" : 1,
                              "self" : true,
                              "lastHeartbeatMessage" : ""
                      },
                      {
                              "_id" : 1,
                              "name" : "34.135.217.111:27017",
                              "health" : 1,
                              "state" : 2,
                              "stateStr" : "SECONDARY",
                              "uptime" : 38,
                              "optime" : {
                                      "ts" : Timestamp(1674043519, 4),
                                      "t" : NumberLong(1)
                              },
                              "optimeDurable" : {
                                      "ts" : Timestamp(1674043519, 4),
                                      "t" : NumberLong(1)
                              },
                              "optimeDate" : ISODate("2023-01-18T12:05:19Z"),
                              "optimeDurableDate" : ISODate("2023-01-18T12:05:19Z"),
                              "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                              "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                              "lastHeartbeat" : ISODate("2023-01-18T12:05:20.649Z"),
                              "lastHeartbeatRecv" : ISODate("2023-01-18T12:05:20.677Z"),
                              "pingMs" : NumberLong(0),
                              "lastHeartbeatMessage" : "",
                              "syncSourceHost" : "34.27.80.226:27017",
                              "syncSourceId" : 0,
                              "infoMessage" : "",
                              "configVersion" : 8,
                              "configTerm" : 1
                      },
                      {
                              "_id" : 2,
                              "name" : "34.66.45.46:27017",
                              "health" : 1,
                              "state" : 2,
                              "stateStr" : "SECONDARY",
                              "uptime" : 44,
                              "optime" : {
                                      "ts" : Timestamp(1674043519, 4),
                                      "t" : NumberLong(1)
                              },
                              "optimeDurable" : {
                                      "ts" : Timestamp(1674043519, 4),
                                      "t" : NumberLong(1)
                              },
                              "optimeDate" : ISODate("2023-01-18T12:05:19Z"),
                              "optimeDurableDate" : ISODate("2023-01-18T12:05:19Z"),
                              "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                              "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                              "lastHeartbeat" : ISODate("2023-01-18T12:05:20.649Z"),
                              "lastHeartbeatRecv" : ISODate("2023-01-18T12:05:21.221Z"),
                              "pingMs" : NumberLong(0),
                              "lastHeartbeatMessage" : "",
                              "syncSourceHost" : "34.135.217.111:27017",
                              "syncSourceId" : 1,
                              "infoMessage" : "",
                              "configVersion" : 8,
                              "configTerm" : 1
                      }
              ],
              "ok" : 1,
              "$clusterTime" : {
                      "clusterTime" : Timestamp(1674043519, 4),
                      "signature" : {
                              "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                              "keyId" : NumberLong(0)
                      }
              },
              "operationTime" : Timestamp(1674043519, 4)
      }
      

      1.13 after:

      NAME                    TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)     AGE
      my-cluster-name-rs0     ClusterIP   None            <none>        27017/TCP   8m25s
      my-cluster-name-rs0-0   ClusterIP   10.58.120.212   <none>        27017/TCP   8m22s
      my-cluster-name-rs0-1   ClusterIP   10.58.115.211   <none>        27017/TCP   7m38s
      my-cluster-name-rs0-2   ClusterIP   10.58.113.110   <none>        27017/TCP   7m2s
      
      NAME              ENDPOINT                                     STATUS         AGE
      my-cluster-name   my-cluster-name-rs0.test.svc.cluster.local   initializing   9m32s
      
      $ psmdb-client.sh
      mongodb_uri: mongodb://backup:[email protected]/admin?ssl=false&authSource=admin&replicaSet=rs0
      If you don't see a command prompt, try pressing enter.
      {"t":{"$date":"2023-01-18T10:02:36.741Z"},"s":"I",  "c":"NETWORK",  "id":4333208, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM host selection timeout","attr":{"replicaSet":"rs0","error":"FailedToSatisfyReadPreference: Could not find host matching read preference { mode: \"nearest\" } for set rs0"}}
      Error: connect failed to replica set rs0/my-cluster-name-rs0-0.test.svc.cluster.local:27017 :
      [email protected]/mongo/shell/mongo.js:374:17
      @(connect):2:6
      exception: connect failed
      exiting with code 1
      pod "percona-client-4946" deleted
      pod test/percona-client-4946 terminated (Error)
      

      1.13 operator log: operator.log

      1.12 before:

      #
      # 1.12 before
      #
      NAME                    TYPE           CLUSTER-IP      EXTERNAL-IP      PORT(S)           AGE
      my-cluster-name-rs0     ClusterIP      None            <none>           27017/TCP         2m34s
      my-cluster-name-rs0-0   LoadBalancer   10.58.113.255   34.136.125.249   27017:31978/TCP   2m31s
      my-cluster-name-rs0-1   LoadBalancer   10.58.124.240   35.194.50.109    27017:30905/TCP   105s
      my-cluster-name-rs0-2   LoadBalancer   10.58.124.61    34.135.119.222   27017:31863/TCP   71s
      
      rs0:PRIMARY> rs.status()
      {
              "set" : "rs0",
              "date" : ISODate("2023-01-18T11:52:08.778Z"),
              "myState" : 1,
              "term" : NumberLong(1),
              "syncSourceHost" : "",
              "syncSourceId" : -1,
              "heartbeatIntervalMillis" : NumberLong(2000),
              "majorityVoteCount" : 2,
              "writeMajorityCount" : 2,
              "votingMembersCount" : 3,
              "writableVotingMembersCount" : 3,
              "optimes" : {
                      "lastCommittedOpTime" : {
                              "ts" : Timestamp(1674042728, 1),
                              "t" : NumberLong(1)
                      },
                      "lastCommittedWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                      "readConcernMajorityOpTime" : {
                              "ts" : Timestamp(1674042728, 1),
                              "t" : NumberLong(1)
                      },
                      "appliedOpTime" : {
                              "ts" : Timestamp(1674042728, 1),
                              "t" : NumberLong(1)
                      },
                      "durableOpTime" : {
                              "ts" : Timestamp(1674042728, 1),
                              "t" : NumberLong(1)
                      },
                      "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                      "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z")
              },
              "lastStableRecoveryTimestamp" : Timestamp(1674042679, 6),
              "electionCandidateMetrics" : {
                      "lastElectionReason" : "electionTimeout",
                      "lastElectionDate" : ISODate("2023-01-18T11:50:23.336Z"),
                      "electionTerm" : NumberLong(1),
                      "lastCommittedOpTimeAtElection" : {
                              "ts" : Timestamp(1674042623, 1),
                              "t" : NumberLong(-1)
                      },
                      "lastSeenOpTimeAtElection" : {
                              "ts" : Timestamp(1674042623, 1),
                              "t" : NumberLong(-1)
                      },
                      "numVotesNeeded" : 1,
                      "priorityAtElection" : 1,
                      "electionTimeoutMillis" : NumberLong(10000),
                      "newTermStartDate" : ISODate("2023-01-18T11:50:23.425Z"),
                      "wMajorityWriteAvailabilityDate" : ISODate("2023-01-18T11:50:23.441Z")
              },
              "members" : [
                      {
                              "_id" : 0,
                              "name" : "34.136.125.249:27017",
                              "health" : 1,
                              "state" : 1,
                              "stateStr" : "PRIMARY",
                              "uptime" : 175,
                              "optime" : {
                                      "ts" : Timestamp(1674042728, 1),
                                      "t" : NumberLong(1)
                              },
                              "optimeDate" : ISODate("2023-01-18T11:52:08Z"),
                              "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                              "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                              "syncSourceHost" : "",
                              "syncSourceId" : -1,
                              "infoMessage" : "Could not find member to sync from",
                              "electionTime" : Timestamp(1674042623, 2),
                              "electionDate" : ISODate("2023-01-18T11:50:23Z"),
                              "configVersion" : 53733,
                              "configTerm" : -1,
                              "self" : true,
                              "lastHeartbeatMessage" : ""
                      },
                      {
                              "_id" : 1,
                              "name" : "35.194.50.109:27017",
                              "health" : 1,
                              "state" : 2,
                              "stateStr" : "SECONDARY",
                              "uptime" : 79,
                              "optime" : {
                                      "ts" : Timestamp(1674042724, 6),
                                      "t" : NumberLong(1)
                              },
                              "optimeDurable" : {
                                      "ts" : Timestamp(1674042724, 6),
                                      "t" : NumberLong(1)
                              },
                              "optimeDate" : ISODate("2023-01-18T11:52:04Z"),
                              "optimeDurableDate" : ISODate("2023-01-18T11:52:04Z"),
                              "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                              "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                              "lastHeartbeat" : ISODate("2023-01-18T11:52:07.740Z"),
                              "lastHeartbeatRecv" : ISODate("2023-01-18T11:52:07.999Z"),
                              "pingMs" : NumberLong(0),
                              "lastHeartbeatMessage" : "",
                              "syncSourceHost" : "34.136.125.249:27017",
                              "syncSourceId" : 0,
                              "infoMessage" : "",
                              "configVersion" : 53733,
                              "configTerm" : -1
                      },
                      {
                              "_id" : 2,
                              "name" : "34.135.119.222:27017",
                              "health" : 1,
                              "state" : 2,
                              "stateStr" : "SECONDARY",
                              "uptime" : 79,
                              "optime" : {
                                      "ts" : Timestamp(1674042724, 6),
                                      "t" : NumberLong(1)
                              },
                              "optimeDurable" : {
                                      "ts" : Timestamp(1674042724, 6),
                                      "t" : NumberLong(1)
                              },
                              "optimeDate" : ISODate("2023-01-18T11:52:04Z"),
                              "optimeDurableDate" : ISODate("2023-01-18T11:52:04Z"),
                              "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                              "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                              "lastHeartbeat" : ISODate("2023-01-18T11:52:07.740Z"),
                              "lastHeartbeatRecv" : ISODate("2023-01-18T11:52:06.895Z"),
                              "pingMs" : NumberLong(0),
                              "lastHeartbeatMessage" : "",
                              "syncSourceHost" : "35.194.50.109:27017",
                              "syncSourceId" : 1,
                              "infoMessage" : "",
                              "configVersion" : 53733,
                              "configTerm" : -1
                      }
              ],
              "ok" : 1,
              "$clusterTime" : {
                      "clusterTime" : Timestamp(1674042728, 1),
                      "signature" : {
                              "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                              "keyId" : NumberLong(0)
                      }
              },
              "operationTime" : Timestamp(1674042728, 1)
      }
      
      rs0:PRIMARY> rs.config()
      {
              "_id" : "rs0",
              "version" : 53733,
              "members" : [
                      {
                              "_id" : 0,
                              "host" : "34.136.125.249:27017",
                              "arbiterOnly" : false,
                              "buildIndexes" : true,
                              "hidden" : false,
                              "priority" : 2,
                              "tags" : {
                                      "podName" : "my-cluster-name-rs0-0",
                                      "serviceName" : "my-cluster-name"
                              },
                              "secondaryDelaySecs" : NumberLong(0),
                              "votes" : 1
                      },
                      {
                              "_id" : 1,
                              "host" : "35.194.50.109:27017",
                              "arbiterOnly" : false,
                              "buildIndexes" : true,
                              "hidden" : false,
                              "priority" : 2,
                              "tags" : {
                                      "podName" : "my-cluster-name-rs0-1",
                                      "serviceName" : "my-cluster-name"
                              },
                              "secondaryDelaySecs" : NumberLong(0),
                              "votes" : 1
                      },
                      {
                              "_id" : 2,
                              "host" : "34.135.119.222:27017",
                              "arbiterOnly" : false,
                              "buildIndexes" : true,
                              "hidden" : false,
                              "priority" : 2,
                              "tags" : {
                                      "podName" : "my-cluster-name-rs0-2",
                                      "serviceName" : "my-cluster-name"
                              },
                              "secondaryDelaySecs" : NumberLong(0),
                              "votes" : 1
                      }
              ],
              "protocolVersion" : NumberLong(1),
              "writeConcernMajorityJournalDefault" : true,
              "settings" : {
                      "chainingAllowed" : true,
                      "heartbeatIntervalMillis" : 2000,
                      "heartbeatTimeoutSecs" : 10,
                      "electionTimeoutMillis" : 10000,
                      "catchUpTimeoutMillis" : -1,
                      "catchUpTakeoverDelayMillis" : 30000,
                      "getLastErrorModes" : {
      
                      },
                      "getLastErrorDefaults" : {
                              "w" : 1,
                              "wtimeout" : 0
                      },
                      "replicaSetId" : ObjectId("63c7dcff2a02a8a4a5e2375d")
              }
      }
      

      1.12 after:

      #
      # 1.12 after
      #
      
      NAME                    TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)     AGE
      my-cluster-name-rs0     ClusterIP   None            <none>        27017/TCP   5m13s
      my-cluster-name-rs0-0   ClusterIP   10.58.113.255   <none>        27017/TCP   5m10s
      my-cluster-name-rs0-1   ClusterIP   10.58.124.240   <none>        27017/TCP   4m24s
      my-cluster-name-rs0-2   ClusterIP   10.58.124.61    <none>        27017/TCP   3m50s
      
      NAME              ENDPOINT                                     STATUS   AGE
      my-cluster-name   my-cluster-name-rs0.test.svc.cluster.local   ready    5m49s
      
      rs0:PRIMARY> rs.status()
      {
              "set" : "rs0",
              "date" : ISODate("2023-01-18T11:55:16.135Z"),
              "myState" : 1,
              "term" : NumberLong(1),
              "syncSourceHost" : "",
              "syncSourceId" : -1,
              "heartbeatIntervalMillis" : NumberLong(2000),
              "majorityVoteCount" : 2,
              "writeMajorityCount" : 2,
              "votingMembersCount" : 3,
              "writableVotingMembersCount" : 3,
              "optimes" : {
                      "lastCommittedOpTime" : {
                              "ts" : Timestamp(1674042915, 1),
                              "t" : NumberLong(1)
                      },
                      "lastCommittedWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                      "readConcernMajorityOpTime" : {
                              "ts" : Timestamp(1674042915, 1),
                              "t" : NumberLong(1)
                      },
                      "appliedOpTime" : {
                              "ts" : Timestamp(1674042915, 1),
                              "t" : NumberLong(1)
                      },
                      "durableOpTime" : {
                              "ts" : Timestamp(1674042915, 1),
                              "t" : NumberLong(1)
                      },
                      "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                      "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z")
              },
              "lastStableRecoveryTimestamp" : Timestamp(1674042857, 1),
              "electionCandidateMetrics" : {
                      "lastElectionReason" : "electionTimeout",
                      "lastElectionDate" : ISODate("2023-01-18T11:50:23.336Z"),
                      "electionTerm" : NumberLong(1),
                      "lastCommittedOpTimeAtElection" : {
                              "ts" : Timestamp(1674042623, 1),
                              "t" : NumberLong(-1)
                      },
                      "lastSeenOpTimeAtElection" : {
                              "ts" : Timestamp(1674042623, 1),
                              "t" : NumberLong(-1)
                      },
                      "numVotesNeeded" : 1,
                      "priorityAtElection" : 1,
                      "electionTimeoutMillis" : NumberLong(10000),
                      "newTermStartDate" : ISODate("2023-01-18T11:50:23.425Z"),
                      "wMajorityWriteAvailabilityDate" : ISODate("2023-01-18T11:50:23.441Z")
              },
              "members" : [
                      {
                              "_id" : 0,
                              "name" : "10.58.113.255:27017",
                              "health" : 1,
                              "state" : 1,
                              "stateStr" : "PRIMARY",
                              "uptime" : 363,
                              "optime" : {
                                      "ts" : Timestamp(1674042915, 1),
                                      "t" : NumberLong(1)
                              },
                              "optimeDate" : ISODate("2023-01-18T11:55:15Z"),
                              "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                              "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                              "syncSourceHost" : "",
                              "syncSourceId" : -1,
                              "infoMessage" : "",
                              "electionTime" : Timestamp(1674042623, 2),
                              "electionDate" : ISODate("2023-01-18T11:50:23Z"),
                              "configVersion" : 202648,
                              "configTerm" : -1,
                              "self" : true,
                              "lastHeartbeatMessage" : ""
                      },
                      {
                              "_id" : 1,
                              "name" : "10.58.124.240:27017",
                              "health" : 1,
                              "state" : 2,
                              "stateStr" : "SECONDARY",
                              "uptime" : 58,
                              "optime" : {
                                      "ts" : Timestamp(1674042915, 1),
                                      "t" : NumberLong(1)
                              },
                              "optimeDurable" : {
                                      "ts" : Timestamp(1674042915, 1),
                                      "t" : NumberLong(1)
                              },
                              "optimeDate" : ISODate("2023-01-18T11:55:15Z"),
                              "optimeDurableDate" : ISODate("2023-01-18T11:55:15Z"),
                              "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                              "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                              "lastHeartbeat" : ISODate("2023-01-18T11:55:15.629Z"),
                              "lastHeartbeatRecv" : ISODate("2023-01-18T11:55:14.451Z"),
                              "pingMs" : NumberLong(0),
                              "lastHeartbeatMessage" : "",
                              "syncSourceHost" : "10.58.113.255:27017",
                              "syncSourceId" : 0,
                              "infoMessage" : "",
                              "configVersion" : 202648,
                              "configTerm" : -1
                      },
                      {
                              "_id" : 2,
                              "name" : "10.58.124.61:27017",
                              "health" : 1,
                              "state" : 2,
                              "stateStr" : "SECONDARY",
                              "uptime" : 58,
                              "optime" : {
                                      "ts" : Timestamp(1674042915, 1),
                                      "t" : NumberLong(1)
                              },
                              "optimeDurable" : {
                                      "ts" : Timestamp(1674042915, 1),
                                      "t" : NumberLong(1)
                              },
                              "optimeDate" : ISODate("2023-01-18T11:55:15Z"),
                              "optimeDurableDate" : ISODate("2023-01-18T11:55:15Z"),
                              "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                              "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                              "lastHeartbeat" : ISODate("2023-01-18T11:55:15.629Z"),
                              "lastHeartbeatRecv" : ISODate("2023-01-18T11:55:14.451Z"),
                              "pingMs" : NumberLong(0),
                              "lastHeartbeatMessage" : "",
                              "syncSourceHost" : "10.58.113.255:27017",
                              "syncSourceId" : 0,
                              "infoMessage" : "",
                              "configVersion" : 202648,
                              "configTerm" : -1
                      }
              ],
              "ok" : 1,
              "$clusterTime" : {
                      "clusterTime" : Timestamp(1674042915, 1),
                      "signature" : {
                              "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                              "keyId" : NumberLong(0)
                      }
              },
              "operationTime" : Timestamp(1674042915, 1)
      }
      
      rs0:PRIMARY> rs.config()
      {
              "_id" : "rs0",
              "version" : 202648,
              "members" : [
                      {
                              "_id" : 0,
                              "host" : "10.58.113.255:27017",
                              "arbiterOnly" : false,
                              "buildIndexes" : true,
                              "hidden" : false,
                              "priority" : 2,
                              "tags" : {
                                      "serviceName" : "my-cluster-name",
                                      "podName" : "my-cluster-name-rs0-0"
                              },
                              "secondaryDelaySecs" : NumberLong(0),
                              "votes" : 1
                      },
                      {
                              "_id" : 1,
                              "host" : "10.58.124.240:27017",
                              "arbiterOnly" : false,
                              "buildIndexes" : true,
                              "hidden" : false,
                              "priority" : 2,
                              "tags" : {
                                      "podName" : "my-cluster-name-rs0-1",
                                      "serviceName" : "my-cluster-name"
                              },
                              "secondaryDelaySecs" : NumberLong(0),
                              "votes" : 1
                      },
                      {
                              "_id" : 2,
                              "host" : "10.58.124.61:27017",
                              "arbiterOnly" : false,
                              "buildIndexes" : true,
                              "hidden" : false,
                              "priority" : 2,
                              "tags" : {
                                      "serviceName" : "my-cluster-name",
                                      "podName" : "my-cluster-name-rs0-2"
                              },
                              "secondaryDelaySecs" : NumberLong(0),
                              "votes" : 1
                      }
              ],
              "protocolVersion" : NumberLong(1),
              "writeConcernMajorityJournalDefault" : true,
              "settings" : {
                      "chainingAllowed" : true,
                      "heartbeatIntervalMillis" : 2000,
                      "heartbeatTimeoutSecs" : 10,
                      "electionTimeoutMillis" : 10000,
                      "catchUpTimeoutMillis" : -1,
                      "catchUpTakeoverDelayMillis" : 30000,
                      "getLastErrorModes" : {
      
                      },
                      "getLastErrorDefaults" : {
                              "w" : 1,
                              "wtimeout" : 0
                      },
                      "replicaSetId" : ObjectId("63c7dcff2a02a8a4a5e2375d")
              }
      }
      

      Attachments

        Activity

          People

            Unassigned Unassigned
            tomislav.plavcic@percona.com Tomislav Plavcic
            Votes:
            0 Vote for this issue
            Watchers:
            6 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Smart Checklist