Uploaded image for project: 'Percona Operator for MongoDB'
  1. Percona Operator for MongoDB
  2. K8SPSMDB-841

Cluster unready after switching from expose LoadBalancer to ClusterIP

    XMLWordPrintable

Details

    • Bug
    • Status: Done
    • Medium
    • Resolution: Fixed
    • 1.13.0
    • 1.14.0
    • None
    • None
    • Yes
    • Yes
    • Yes

    Description

      As reported on slack, if you run replica set with 1.12 and expose set to LoadBalancer for replica nodes and you switch to ClusterIP later the replica will be reconfigured with these new IPs.

      With 1.13 it's not the case, but the cluster is left in initializing state.
      I think it's possible that in 1.12 it worked because we had force option enabled on replica reconfigure, but we have removed it because of other problems this creates.

      Here's how it looks:
      1.13 before:

      NAME                    TYPE           CLUSTER-IP      EXTERNAL-IP      PORT(S)           AGE
my-cluster-name-rs0     ClusterIP      None            <none>           27017/TCP         119s
my-cluster-name-rs0-0   LoadBalancer   10.58.120.212   34.27.80.226     27017:31110/TCP   116s
my-cluster-name-rs0-1   LoadBalancer   10.58.115.211   34.135.217.111   27017:30439/TCP   72s
my-cluster-name-rs0-2   LoadBalancer   10.58.113.110   34.66.45.46      27017:31794/TCP   36s

rs0:PRIMARY> rs.config()
{
        "_id" : "rs0",
        "version" : 8,
        "term" : 1,
        "members" : [
                {
                        "_id" : 0,
                        "host" : "34.27.80.226:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 2,
                        "tags" : {
                                "podName" : "my-cluster-name-rs0-0",
                                "serviceName" : "my-cluster-name"
                        },
                        "secondaryDelaySecs" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 1,
                        "host" : "34.135.217.111:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 2,
                        "tags" : {
                                "serviceName" : "my-cluster-name",
                                "podName" : "my-cluster-name-rs0-1"
                        },
                        "secondaryDelaySecs" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 2,
                        "host" : "34.66.45.46:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 2,
                        "tags" : {
                                "podName" : "my-cluster-name-rs0-2",
                                "serviceName" : "my-cluster-name"
                        },
                        "secondaryDelaySecs" : NumberLong(0),
                        "votes" : 1
                }
        ],
        "protocolVersion" : NumberLong(1),
        "writeConcernMajorityJournalDefault" : true,
        "settings" : {
                "chainingAllowed" : true,
                "heartbeatIntervalMillis" : 2000,
                "heartbeatTimeoutSecs" : 10,
                "electionTimeoutMillis" : 10000,
                "catchUpTimeoutMillis" : -1,
                "catchUpTakeoverDelayMillis" : 30000,
                "getLastErrorModes" : {

                },
                "getLastErrorDefaults" : {
                        "w" : 1,
                        "wtimeout" : 0
                },
                "replicaSetId" : ObjectId("63c7e02e0f7487651a4cd4d8")
        }
}

rs0:PRIMARY> rs.status()
{
        "set" : "rs0",
        "date" : ISODate("2023-01-18T12:05:21.338Z"),
        "myState" : 1,
        "term" : NumberLong(1),
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "heartbeatIntervalMillis" : NumberLong(2000),
        "majorityVoteCount" : 2,
        "writeMajorityCount" : 2,
        "votingMembersCount" : 3,
        "writableVotingMembersCount" : 3,
        "optimes" : {
                "lastCommittedOpTime" : {
                        "ts" : Timestamp(1674043519, 4),
                        "t" : NumberLong(1)
                },
                "lastCommittedWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                "readConcernMajorityOpTime" : {
                        "ts" : Timestamp(1674043519, 4),
                        "t" : NumberLong(1)
                },
                "appliedOpTime" : {
                        "ts" : Timestamp(1674043519, 4),
                        "t" : NumberLong(1)
                },
                "durableOpTime" : {
                        "ts" : Timestamp(1674043519, 4),
                        "t" : NumberLong(1)
                },
                "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z")
        },
        "lastStableRecoveryTimestamp" : Timestamp(1674043494, 2),
        "electionCandidateMetrics" : {
                "lastElectionReason" : "electionTimeout",
                "lastElectionDate" : ISODate("2023-01-18T12:03:58.335Z"),
                "electionTerm" : NumberLong(1),
                "lastCommittedOpTimeAtElection" : {
                        "ts" : Timestamp(1674043438, 1),
                        "t" : NumberLong(-1)
                },
                "lastSeenOpTimeAtElection" : {
                        "ts" : Timestamp(1674043438, 1),
                        "t" : NumberLong(-1)
                },
                "numVotesNeeded" : 1,
                "priorityAtElection" : 1,
                "electionTimeoutMillis" : NumberLong(10000),
                "newTermStartDate" : ISODate("2023-01-18T12:03:58.414Z"),
                "wMajorityWriteAvailabilityDate" : ISODate("2023-01-18T12:03:58.433Z")
        },
        "members" : [
                {
                        "_id" : 0,
                        "name" : "34.27.80.226:27017",
                        "health" : 1,
                        "state" : 1,
                        "stateStr" : "PRIMARY",
                        "uptime" : 166,
                        "optime" : {
                                "ts" : Timestamp(1674043519, 4),
                                "t" : NumberLong(1)
                        },
                        "optimeDate" : ISODate("2023-01-18T12:05:19Z"),
                        "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                        "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "Could not find member to sync from",
                        "electionTime" : Timestamp(1674043438, 2),
                        "electionDate" : ISODate("2023-01-18T12:03:58Z"),
                        "configVersion" : 8,
                        "configTerm" : 1,
                        "self" : true,
                        "lastHeartbeatMessage" : ""
                },
                {
                        "_id" : 1,
                        "name" : "34.135.217.111:27017",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 38,
                        "optime" : {
                                "ts" : Timestamp(1674043519, 4),
                                "t" : NumberLong(1)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1674043519, 4),
                                "t" : NumberLong(1)
                        },
                        "optimeDate" : ISODate("2023-01-18T12:05:19Z"),
                        "optimeDurableDate" : ISODate("2023-01-18T12:05:19Z"),
                        "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                        "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                        "lastHeartbeat" : ISODate("2023-01-18T12:05:20.649Z"),
                        "lastHeartbeatRecv" : ISODate("2023-01-18T12:05:20.677Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "34.27.80.226:27017",
                        "syncSourceId" : 0,
                        "infoMessage" : "",
                        "configVersion" : 8,
                        "configTerm" : 1
                },
                {
                        "_id" : 2,
                        "name" : "34.66.45.46:27017",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 44,
                        "optime" : {
                                "ts" : Timestamp(1674043519, 4),
                                "t" : NumberLong(1)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1674043519, 4),
                                "t" : NumberLong(1)
                        },
                        "optimeDate" : ISODate("2023-01-18T12:05:19Z"),
                        "optimeDurableDate" : ISODate("2023-01-18T12:05:19Z"),
                        "lastAppliedWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                        "lastDurableWallTime" : ISODate("2023-01-18T12:05:19.163Z"),
                        "lastHeartbeat" : ISODate("2023-01-18T12:05:20.649Z"),
                        "lastHeartbeatRecv" : ISODate("2023-01-18T12:05:21.221Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "34.135.217.111:27017",
                        "syncSourceId" : 1,
                        "infoMessage" : "",
                        "configVersion" : 8,
                        "configTerm" : 1
                }
        ],
        "ok" : 1,
        "$clusterTime" : {
                "clusterTime" : Timestamp(1674043519, 4),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        },
        "operationTime" : Timestamp(1674043519, 4)
}

      1.13 after:

      NAME                    TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)     AGE
my-cluster-name-rs0     ClusterIP   None            <none>        27017/TCP   8m25s
my-cluster-name-rs0-0   ClusterIP   10.58.120.212   <none>        27017/TCP   8m22s
my-cluster-name-rs0-1   ClusterIP   10.58.115.211   <none>        27017/TCP   7m38s
my-cluster-name-rs0-2   ClusterIP   10.58.113.110   <none>        27017/TCP   7m2s

NAME              ENDPOINT                                     STATUS         AGE
my-cluster-name   my-cluster-name-rs0.test.svc.cluster.local   initializing   9m32s

$ psmdb-client.sh
mongodb_uri: mongodb://backup:[email protected]/admin?ssl=false&authSource=admin&replicaSet=rs0
If you don't see a command prompt, try pressing enter.
{"t":{"$date":"2023-01-18T10:02:36.741Z"},"s":"I",  "c":"NETWORK",  "id":4333208, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM host selection timeout","attr":{"replicaSet":"rs0","error":"FailedToSatisfyReadPreference: Could not find host matching read preference { mode: \"nearest\" } for set rs0"}}
Error: connect failed to replica set rs0/my-cluster-name-rs0-0.test.svc.cluster.local:27017 :
[email protected]/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
pod "percona-client-4946" deleted
pod test/percona-client-4946 terminated (Error)

      1.13 operator log: operator.log

      1.12 before:

      #
# 1.12 before
#
NAME                    TYPE           CLUSTER-IP      EXTERNAL-IP      PORT(S)           AGE
my-cluster-name-rs0     ClusterIP      None            <none>           27017/TCP         2m34s
my-cluster-name-rs0-0   LoadBalancer   10.58.113.255   34.136.125.249   27017:31978/TCP   2m31s
my-cluster-name-rs0-1   LoadBalancer   10.58.124.240   35.194.50.109    27017:30905/TCP   105s
my-cluster-name-rs0-2   LoadBalancer   10.58.124.61    34.135.119.222   27017:31863/TCP   71s

rs0:PRIMARY> rs.status()
{
        "set" : "rs0",
        "date" : ISODate("2023-01-18T11:52:08.778Z"),
        "myState" : 1,
        "term" : NumberLong(1),
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "heartbeatIntervalMillis" : NumberLong(2000),
        "majorityVoteCount" : 2,
        "writeMajorityCount" : 2,
        "votingMembersCount" : 3,
        "writableVotingMembersCount" : 3,
        "optimes" : {
                "lastCommittedOpTime" : {
                        "ts" : Timestamp(1674042728, 1),
                        "t" : NumberLong(1)
                },
                "lastCommittedWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                "readConcernMajorityOpTime" : {
                        "ts" : Timestamp(1674042728, 1),
                        "t" : NumberLong(1)
                },
                "appliedOpTime" : {
                        "ts" : Timestamp(1674042728, 1),
                        "t" : NumberLong(1)
                },
                "durableOpTime" : {
                        "ts" : Timestamp(1674042728, 1),
                        "t" : NumberLong(1)
                },
                "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z")
        },
        "lastStableRecoveryTimestamp" : Timestamp(1674042679, 6),
        "electionCandidateMetrics" : {
                "lastElectionReason" : "electionTimeout",
                "lastElectionDate" : ISODate("2023-01-18T11:50:23.336Z"),
                "electionTerm" : NumberLong(1),
                "lastCommittedOpTimeAtElection" : {
                        "ts" : Timestamp(1674042623, 1),
                        "t" : NumberLong(-1)
                },
                "lastSeenOpTimeAtElection" : {
                        "ts" : Timestamp(1674042623, 1),
                        "t" : NumberLong(-1)
                },
                "numVotesNeeded" : 1,
                "priorityAtElection" : 1,
                "electionTimeoutMillis" : NumberLong(10000),
                "newTermStartDate" : ISODate("2023-01-18T11:50:23.425Z"),
                "wMajorityWriteAvailabilityDate" : ISODate("2023-01-18T11:50:23.441Z")
        },
        "members" : [
                {
                        "_id" : 0,
                        "name" : "34.136.125.249:27017",
                        "health" : 1,
                        "state" : 1,
                        "stateStr" : "PRIMARY",
                        "uptime" : 175,
                        "optime" : {
                                "ts" : Timestamp(1674042728, 1),
                                "t" : NumberLong(1)
                        },
                        "optimeDate" : ISODate("2023-01-18T11:52:08Z"),
                        "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                        "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "Could not find member to sync from",
                        "electionTime" : Timestamp(1674042623, 2),
                        "electionDate" : ISODate("2023-01-18T11:50:23Z"),
                        "configVersion" : 53733,
                        "configTerm" : -1,
                        "self" : true,
                        "lastHeartbeatMessage" : ""
                },
                {
                        "_id" : 1,
                        "name" : "35.194.50.109:27017",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 79,
                        "optime" : {
                                "ts" : Timestamp(1674042724, 6),
                                "t" : NumberLong(1)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1674042724, 6),
                                "t" : NumberLong(1)
                        },
                        "optimeDate" : ISODate("2023-01-18T11:52:04Z"),
                        "optimeDurableDate" : ISODate("2023-01-18T11:52:04Z"),
                        "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                        "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                        "lastHeartbeat" : ISODate("2023-01-18T11:52:07.740Z"),
                        "lastHeartbeatRecv" : ISODate("2023-01-18T11:52:07.999Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "34.136.125.249:27017",
                        "syncSourceId" : 0,
                        "infoMessage" : "",
                        "configVersion" : 53733,
                        "configTerm" : -1
                },
                {
                        "_id" : 2,
                        "name" : "34.135.119.222:27017",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 79,
                        "optime" : {
                                "ts" : Timestamp(1674042724, 6),
                                "t" : NumberLong(1)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1674042724, 6),
                                "t" : NumberLong(1)
                        },
                        "optimeDate" : ISODate("2023-01-18T11:52:04Z"),
                        "optimeDurableDate" : ISODate("2023-01-18T11:52:04Z"),
                        "lastAppliedWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                        "lastDurableWallTime" : ISODate("2023-01-18T11:52:08.708Z"),
                        "lastHeartbeat" : ISODate("2023-01-18T11:52:07.740Z"),
                        "lastHeartbeatRecv" : ISODate("2023-01-18T11:52:06.895Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "35.194.50.109:27017",
                        "syncSourceId" : 1,
                        "infoMessage" : "",
                        "configVersion" : 53733,
                        "configTerm" : -1
                }
        ],
        "ok" : 1,
        "$clusterTime" : {
                "clusterTime" : Timestamp(1674042728, 1),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        },
        "operationTime" : Timestamp(1674042728, 1)
}

rs0:PRIMARY> rs.config()
{
        "_id" : "rs0",
        "version" : 53733,
        "members" : [
                {
                        "_id" : 0,
                        "host" : "34.136.125.249:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 2,
                        "tags" : {
                                "podName" : "my-cluster-name-rs0-0",
                                "serviceName" : "my-cluster-name"
                        },
                        "secondaryDelaySecs" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 1,
                        "host" : "35.194.50.109:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 2,
                        "tags" : {
                                "podName" : "my-cluster-name-rs0-1",
                                "serviceName" : "my-cluster-name"
                        },
                        "secondaryDelaySecs" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 2,
                        "host" : "34.135.119.222:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 2,
                        "tags" : {
                                "podName" : "my-cluster-name-rs0-2",
                                "serviceName" : "my-cluster-name"
                        },
                        "secondaryDelaySecs" : NumberLong(0),
                        "votes" : 1
                }
        ],
        "protocolVersion" : NumberLong(1),
        "writeConcernMajorityJournalDefault" : true,
        "settings" : {
                "chainingAllowed" : true,
                "heartbeatIntervalMillis" : 2000,
                "heartbeatTimeoutSecs" : 10,
                "electionTimeoutMillis" : 10000,
                "catchUpTimeoutMillis" : -1,
                "catchUpTakeoverDelayMillis" : 30000,
                "getLastErrorModes" : {

                },
                "getLastErrorDefaults" : {
                        "w" : 1,
                        "wtimeout" : 0
                },
                "replicaSetId" : ObjectId("63c7dcff2a02a8a4a5e2375d")
        }
}

      1.12 after:

      #
# 1.12 after
#

NAME                    TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)     AGE
my-cluster-name-rs0     ClusterIP   None            <none>        27017/TCP   5m13s
my-cluster-name-rs0-0   ClusterIP   10.58.113.255   <none>        27017/TCP   5m10s
my-cluster-name-rs0-1   ClusterIP   10.58.124.240   <none>        27017/TCP   4m24s
my-cluster-name-rs0-2   ClusterIP   10.58.124.61    <none>        27017/TCP   3m50s

NAME              ENDPOINT                                     STATUS   AGE
my-cluster-name   my-cluster-name-rs0.test.svc.cluster.local   ready    5m49s

rs0:PRIMARY> rs.status()
{
        "set" : "rs0",
        "date" : ISODate("2023-01-18T11:55:16.135Z"),
        "myState" : 1,
        "term" : NumberLong(1),
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "heartbeatIntervalMillis" : NumberLong(2000),
        "majorityVoteCount" : 2,
        "writeMajorityCount" : 2,
        "votingMembersCount" : 3,
        "writableVotingMembersCount" : 3,
        "optimes" : {
                "lastCommittedOpTime" : {
                        "ts" : Timestamp(1674042915, 1),
                        "t" : NumberLong(1)
                },
                "lastCommittedWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                "readConcernMajorityOpTime" : {
                        "ts" : Timestamp(1674042915, 1),
                        "t" : NumberLong(1)
                },
                "appliedOpTime" : {
                        "ts" : Timestamp(1674042915, 1),
                        "t" : NumberLong(1)
                },
                "durableOpTime" : {
                        "ts" : Timestamp(1674042915, 1),
                        "t" : NumberLong(1)
                },
                "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z")
        },
        "lastStableRecoveryTimestamp" : Timestamp(1674042857, 1),
        "electionCandidateMetrics" : {
                "lastElectionReason" : "electionTimeout",
                "lastElectionDate" : ISODate("2023-01-18T11:50:23.336Z"),
                "electionTerm" : NumberLong(1),
                "lastCommittedOpTimeAtElection" : {
                        "ts" : Timestamp(1674042623, 1),
                        "t" : NumberLong(-1)
                },
                "lastSeenOpTimeAtElection" : {
                        "ts" : Timestamp(1674042623, 1),
                        "t" : NumberLong(-1)
                },
                "numVotesNeeded" : 1,
                "priorityAtElection" : 1,
                "electionTimeoutMillis" : NumberLong(10000),
                "newTermStartDate" : ISODate("2023-01-18T11:50:23.425Z"),
                "wMajorityWriteAvailabilityDate" : ISODate("2023-01-18T11:50:23.441Z")
        },
        "members" : [
                {
                        "_id" : 0,
                        "name" : "10.58.113.255:27017",
                        "health" : 1,
                        "state" : 1,
                        "stateStr" : "PRIMARY",
                        "uptime" : 363,
                        "optime" : {
                                "ts" : Timestamp(1674042915, 1),
                                "t" : NumberLong(1)
                        },
                        "optimeDate" : ISODate("2023-01-18T11:55:15Z"),
                        "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                        "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "electionTime" : Timestamp(1674042623, 2),
                        "electionDate" : ISODate("2023-01-18T11:50:23Z"),
                        "configVersion" : 202648,
                        "configTerm" : -1,
                        "self" : true,
                        "lastHeartbeatMessage" : ""
                },
                {
                        "_id" : 1,
                        "name" : "10.58.124.240:27017",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 58,
                        "optime" : {
                                "ts" : Timestamp(1674042915, 1),
                                "t" : NumberLong(1)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1674042915, 1),
                                "t" : NumberLong(1)
                        },
                        "optimeDate" : ISODate("2023-01-18T11:55:15Z"),
                        "optimeDurableDate" : ISODate("2023-01-18T11:55:15Z"),
                        "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                        "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                        "lastHeartbeat" : ISODate("2023-01-18T11:55:15.629Z"),
                        "lastHeartbeatRecv" : ISODate("2023-01-18T11:55:14.451Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "10.58.113.255:27017",
                        "syncSourceId" : 0,
                        "infoMessage" : "",
                        "configVersion" : 202648,
                        "configTerm" : -1
                },
                {
                        "_id" : 2,
                        "name" : "10.58.124.61:27017",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 58,
                        "optime" : {
                                "ts" : Timestamp(1674042915, 1),
                                "t" : NumberLong(1)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1674042915, 1),
                                "t" : NumberLong(1)
                        },
                        "optimeDate" : ISODate("2023-01-18T11:55:15Z"),
                        "optimeDurableDate" : ISODate("2023-01-18T11:55:15Z"),
                        "lastAppliedWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                        "lastDurableWallTime" : ISODate("2023-01-18T11:55:15.570Z"),
                        "lastHeartbeat" : ISODate("2023-01-18T11:55:15.629Z"),
                        "lastHeartbeatRecv" : ISODate("2023-01-18T11:55:14.451Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "10.58.113.255:27017",
                        "syncSourceId" : 0,
                        "infoMessage" : "",
                        "configVersion" : 202648,
                        "configTerm" : -1
                }
        ],
        "ok" : 1,
        "$clusterTime" : {
                "clusterTime" : Timestamp(1674042915, 1),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        },
        "operationTime" : Timestamp(1674042915, 1)
}

rs0:PRIMARY> rs.config()
{
        "_id" : "rs0",
        "version" : 202648,
        "members" : [
                {
                        "_id" : 0,
                        "host" : "10.58.113.255:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 2,
                        "tags" : {
                                "serviceName" : "my-cluster-name",
                                "podName" : "my-cluster-name-rs0-0"
                        },
                        "secondaryDelaySecs" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 1,
                        "host" : "10.58.124.240:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 2,
                        "tags" : {
                                "podName" : "my-cluster-name-rs0-1",
                                "serviceName" : "my-cluster-name"
                        },
                        "secondaryDelaySecs" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 2,
                        "host" : "10.58.124.61:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 2,
                        "tags" : {
                                "serviceName" : "my-cluster-name",
                                "podName" : "my-cluster-name-rs0-2"
                        },
                        "secondaryDelaySecs" : NumberLong(0),
                        "votes" : 1
                }
        ],
        "protocolVersion" : NumberLong(1),
        "writeConcernMajorityJournalDefault" : true,
        "settings" : {
                "chainingAllowed" : true,
                "heartbeatIntervalMillis" : 2000,
                "heartbeatTimeoutSecs" : 10,
                "electionTimeoutMillis" : 10000,
                "catchUpTimeoutMillis" : -1,
                "catchUpTakeoverDelayMillis" : 30000,
                "getLastErrorModes" : {

                },
                "getLastErrorDefaults" : {
                        "w" : 1,
                        "wtimeout" : 0
                },
                "replicaSetId" : ObjectId("63c7dcff2a02a8a4a5e2375d")
        }
}

      Attachments

        Activity

          People

            Unassigned Unassigned
            tomislav.plavcic@percona.com Tomislav Plavcic
            Votes:
            0 Vote for this issue
            Watchers:
            6 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Smart Checklist