Failed to downscale/upscale cluster to unsafe configuration

2022-09-14T18:55:17.776Z	ERROR	controller.psmdb-controller	Reconciler error	{"name": "my-cluster-name", "namespace": "psmdb", "error": "reconcile StatefulSet for cfg: failed to run smartUpdate: failed to check active jobs: getting pbm object: create PBM connection to my-cluster-name-rs0-0.my-cluster-name-rs0.psmdb.svc.cluster.local:27017,my-cluster-name-rs0-1.my-cluster-name-rs0.psmdb.svc.cluster.local:27017: create mongo connection to configsvr with connection string 'mongodb://backup:[email protected].svc.cluster.local:27017,my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb.svc.cluster.local:27017,my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb.svc.cluster.local:27017/': mongo ping: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: my-cluster-name-cfg-0.my-cluster-name-cfg.psmdb.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=my-cluster-name,podName=my-cluster-name-cfg-0, Average RTT: 1091936 }, { Addr: my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb.svc.cluster.local:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb.svc.cluster.local on 10.96.0.10:53: no such host }, { Addr: my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb.svc.cluster.local:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb.svc.cluster.local on 10.96.0.10:53: no such host }, ] }", "errorVerbose": "reconcile StatefulSet for cfg: failed to run smartUpdate: failed to check active jobs: getting pbm object: create PBM connection to my-cluster-name-rs0-0.my-cluster-name-rs0.psmdb.svc.cluster.local:27017,my-cluster-name-rs0-1.my-cluster-name-rs0.psmdb.svc.cluster.local:27017: create mongo connection to configsvr with connection string 'mongodb://backup:[email protected].svc.cluster.local:27017,my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb.svc.cluster.local:27017,my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb.svc.cluster.local:27017/': mongo ping: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: my-cluster-name-cfg-0.my-cluster-name-cfg.psmdb.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=my-cluster-name,podName=my-cluster-name-cfg-0, Average RTT: 1091936 }, { Addr: my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb.svc.cluster.local:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb.svc.cluster.local on 10.96.0.10:53: no such host }, { Addr: my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb.svc.cluster.local:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb.svc.cluster.local on 10.96.0.10:53: no such host }, ] }\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:415\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:227\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1571"}

2022-09-14T18:49:55.400Z	ERROR	controller_psmdb	failed to reconcile cluster	{"Request.Namespace": "psmdb", "Request.Name": "my-cluster-name", "replset": "rs0", "error": "add new: write mongo config: replSetReconfig: (NodeNotFound) Quorum check failed because not enough voting nodes responded; required 2 but only the following 1 voting nodes responded: my-cluster-name-rs0-0.my-cluster-name-rs0.psmdb.svc.cluster.local:27017; the following nodes did not respond affirmatively: my-cluster-name-rs0-2.my-cluster-name-rs0.psmdb.svc.cluster.local:27017 failed with Error connecting to my-cluster-name-rs0-2.my-cluster-name-rs0.psmdb.svc.cluster.local:27017 :: caused by :: Could not find address for my-cluster-name-rs0-2.my-cluster-name-rs0.psmdb.svc.cluster.local:27017: SocketException: Host not found (authoritative)", "errorVerbose": "(NodeNotFound) Quorum check failed because not enough voting nodes responded; required 2 but only the following 1 voting nodes responded: my-cluster-name-rs0-0.my-cluster-name-rs0.psmdb.svc.cluster.local:27017; the following nodes did not respond affirmatively: my-cluster-name-rs0-2.my-cluster-name-rs0.psmdb.svc.cluster.local:27017 failed with Error connecting to my-cluster-name-rs0-2.my-cluster-name-rs0.psmdb.svc.cluster.local:27017 :: caused by :: Could not find address for my-cluster-name-rs0-2.my-cluster-name-rs0.psmdb.svc.cluster.local:27017: SocketException: Host not found (authoritative)\nreplSetReconfig\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.WriteConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:248\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:330\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:499\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:227\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1571\nadd new: write mongo config\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:332\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:499\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:227\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1571"}

[email protected] percona-server-mongodb-operator [release-1.13.0] $ kubectl get pods
NAME                                               READY   STATUS    RESTARTS        AGE
my-cluster-name-cfg-0                              2/2     Running   0               8m16s
my-cluster-name-mongos-0                           1/1     Running   0               6m14s
my-cluster-name-mongos-1                           1/1     Running   0               5m58s
my-cluster-name-mongos-2                           1/1     Running   0               5m44s
my-cluster-name-rs0-0                              2/2     Running   0               8m15s
my-cluster-name-rs0-1                              2/2     Running   0               7m30s
my-cluster-name-rs1-0                              2/2     Running   0               4m20s
percona-server-mongodb-operator-6677c8cbf7-t6z8v   1/1     Running   1 (8m32s ago)   8m46s

2022-09-15T06:29:28.790Z	ERROR	controller.psmdb-controller	Reconciler error	{"name": "my-cluster-name", "namespace": "psmdb", "error": "check remove posibility for rs rs1: dial:: failed to get TLS config: get ssl certSecret: Secret \"my-cluster-name-ssl\" not found", "errorVerbose": "Secret \"my-cluster-name-ssl\" not found\nget ssl certSecret\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/tls.Config\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/tls/tls.go:123\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:39\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:21\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).checkIfPossibleToRemove\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:675\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:227\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1571\nfailed to get TLS config\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:41\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:21\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).checkIfPossibleToRemove\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:675\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:227\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1571\ndial:\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).checkIfPossibleToRemove\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:677\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:227\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1571\ncheck remove posibility for rs rs1\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:323\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:227\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1571"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem

	apply changes to secondary pod	{"pod": "my-cluster-name-cfg-2"}
2022-09-15T07:22:35.023Z	ERROR	controller.psmdb-controller	Reconciler error	{"name": "my-cluster-name", "namespace": "psmdb-1", "error": "reconcile StatefulSet for cfg: failed to run smartUpdate: failed to apply changes: wait pod restart: reach pod wait limit", "errorVerbose": "reconcile StatefulSet for cfg: failed to run smartUpdate: failed to apply changes: wait pod restart: reach pod wait limit\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:415\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:227\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1571"}



2022-09-15T07:24:35.219Z	ERROR	controller.psmdb-controller	Reconciler error	{"name": "my-cluster-name", "namespace": "psmdb-1", "error": "reconcile StatefulSet for cfg: failed to run smartUpdate: failed to check active jobs: getting pbm object: create PBM connection to my-cluster-name-rs0-0.my-cluster-name-rs0.psmdb-1.svc.cluster.local:27017,my-cluster-name-rs0-1.my-cluster-name-rs0.psmdb-1.svc.cluster.local:27017: create mongo connection to configsvr with connection string 'mongodb://backup:[email protected]-1.svc.cluster.local:27017,my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb-1.svc.cluster.local:27017,my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb-1.svc.cluster.local:27017/': mongo ping: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: my-cluster-name-cfg-0.my-cluster-name-cfg.psmdb-1.svc.cluster.local:27017, Type: RSSecondary, Tag sets: podName=my-cluster-name-cfg-0,serviceName=my-cluster-name, Average RTT: 611937 }, { Addr: my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb-1.svc.cluster.local:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb-1.svc.cluster.local on 10.96.0.10:53: no such host }, { Addr: my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb-1.svc.cluster.local:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb-1.svc.cluster.local on 10.96.0.10:53: no such host }, ] }", "errorVerbose": "reconcile StatefulSet for cfg: failed to run smartUpdate: failed to check active jobs: getting pbm object: create PBM connection to my-cluster-name-rs0-0.my-cluster-name-rs0.psmdb-1.svc.cluster.local:27017,my-cluster-name-rs0-1.my-cluster-name-rs0.psmdb-1.svc.cluster.local:27017: create mongo connection to configsvr with connection string 'mongodb://backup:[email protected]-1.svc.cluster.local:27017,my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb-1.svc.cluster.local:27017,my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb-1.svc.cluster.local:27017/': mongo ping: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: my-cluster-name-cfg-0.my-cluster-name-cfg.psmdb-1.svc.cluster.local:27017, Type: RSSecondary, Tag sets: podName=my-cluster-name-cfg-0,serviceName=my-cluster-name, Average RTT: 611937 }, { Addr: my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb-1.svc.cluster.local:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup my-cluster-name-cfg-1.my-cluster-name-cfg.psmdb-1.svc.cluster.local on 10.96.0.10:53: no such host }, { Addr: my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb-1.svc.cluster.local:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp: lookup my-cluster-name-cfg-2.my-cluster-name-cfg.psmdb-1.svc.cluster.local on 10.96.0.10:53: no such host }, ] }\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:415\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/src/github.com/percona/percona-server-mongodb-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:227\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1571"}