使用 Percona Operator for MongoDB 部署 AKS 上的 MongoDB 群集并在群集上安装用来与 MongoDB 交互的 mongo-express
后,可以在多种场景中测试 MongoDB 群集的复原能力。 本教程基于以前的指南中介绍的 MongoDB 群集部署和 MongoDB 群集验证。
生成并运行 MongoDB 的示例客户端应用程序
以下步骤演示如何生成 MongoDB 的示例客户端应用程序,并将应用程序的 Docker 映像推送到 Azure 容器注册表。 示例客户端应用程序使用 Locust 负载测试框架来模拟 MongoDB 群集上的工作负荷。 此应用程序将文档插入 MongoDB 群集并度量每个文档插入操作所花的时间。 此应用程序使用 Faker
库为员工文档生成随机姓名和部门。 可以扩展此类以实现更复杂的操作。
使用以下命令创建新目录:
mkdir mongodb-locust-client cd mongodb-locust-client
使用以下命令在
requirements.txt
目录中创建mongodb-locust-client
文件:cat > requirements.txt <<EOF locust pymongo Faker EOF
使用以下命令在
locustfile.py
目录中创建mongodb-locust-client
:cat > locustfile.py <<EOF import os import random import logging import time import uuid from locust import User, task, constant_throughput, events, tag, between from pymongo import MongoClient from pymongo.errors import ServerSelectionTimeoutError from pymongo.errors import ConnectionFailure from faker import Faker # Set up Faker for generating random names and departments fake = Faker() # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) # List of department names for random selection department_keywords = [ "Accounting", "Marketing", "Human Resources", "Sales", "Research and Development", "IT", "Customer Support", "Legal", "Operations", "Finance", "Product Management", "Public Relations", "Engineering", "Data Science", "Business Development" ] class MongoDBLocust(User): # Set constant throughput (requests per second) wait_time = constant_throughput(20) # MongoDB connection details (make sure these are set correctly in your environment) host = "${MY_CLUSTER_NAME}-mongodb-mongos.mongodb.svc.cluster.local" port = "27017" username = os.getenv('MONGODB_USERNAME') password = os.getenv('MONGODB_PASSWORD') def __init__(self, *args, **kwargs): super(MongoDBLocust, self).__init__(*args, **kwargs) self.client = MongoDBClient(self.username, self.password) # Initialize MongoDB client def on_stop(self): """Called when the simulated user stops, to close the MongoDB client.""" self.client.close() @task @tag("insert") def insert_document(self): """Task to insert a document into MongoDB.""" self.client.insert_document("insert_document") class MongoDBClient(object): """MongoDB client for managing database interactions.""" def __init__(self, username, password): # Set MongoDB connection parameters self.host = "${MY_CLUSTER_NAME}-mongodb-mongos.mongodb.svc.cluster.local" self.port = "27017" self.username = username self.password = password self.client = MongoClient(f"mongodb://{self.username}:{self.password}@{self.host}:{self.port}/") self.db = self.client['employeeDB'] self.collection = self.db['employees'] self.document_id = None # Placeholder for the inserted document ID def insert_document(self, task_name, command='INSERT'): """Insert a document into MongoDB and measure the time.""" employee_id = str(uuid.uuid4()) start_time = time.time() logger.info(f"{employee_id} - Start Time -> start_time: {start_time}") try: # Create a new fake employee document new_document = { 'Id': str(uuid.uuid4()), 'EmployeeId': employee_id, 'FirstName': fake.first_name(), 'LastName': fake.last_name(), 'Department': random.choice(department_keywords) } # Insert the document result = self.collection.insert_one(new_document) self.document_id = str(result.inserted_id) # Store the document ID for future reads # Measure the time taken for the MongoDB operation (in milliseconds) total_time = (time.time() - start_time) * 1000 # Convert to ms logger.info(f"{employee_id} - Task - {task_name} -> total_time: {total_time} ms") # Fire the event to report the request events.request.fire( request_type=command, name=task_name, response_time=total_time, response_length=len(str(result)) # Length of the result returned by the operation ) except Exception as e: total_time = (time.time() - start_time) * 1000 # Time taken with error logger.error(f"{employee_id} - Error in {task_name}: {e} -> total_time: {total_time} ms") events.request.fire( request_type=command, name=task_name, response_time=total_time, response_length=0, # No response in case of error exception=e ) logger.info(f"{employee_id} - End Time -> end time: {time.time()}") def close(self): """Close the MongoDB connection.""" self.client.close() EOF
使用以下命令在
mongodb-locust-client
目录中创建 Dockerfile。 Dockerfile 为示例客户端应用程序创建一个 Docker 映像,使用python:3.10-slim-bullseye
映像作为基础映像,并为应用程序安装所需的依赖项。cat > Dockerfile <<EOF # Use a slim Python image as the base FROM python:3.10-slim-bullseye # Install wget and gnupg RUN apt-get update && apt-get install -y wget gnupg # Add MongoDB GPG Key and repository RUN wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | tee /etc/apt/trusted.gpg.d/mongodb.asc && \ echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/debian bullseye/mongodb-org/6.0 main" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list # Update APT sources and install MongoDB Shell (mongosh) RUN apt-get update && apt-get install -y mongodb-mongosh # Copy your application files COPY requirements.txt ./ COPY locustfile.py ./ # Install Python dependencies RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt EOF
使用
az acr build
命令为示例客户端应用程序生成 Docker 映像。 确保将$MY_ACR_REGISTRY
替换为 Azure 容器注册表的名称。az acr build --image mongodb-locust-client --registry $MY_ACR_REGISTRY .
测试 AKS 上的 MongoDB 群集
创建 MongoDB Locust 客户端 Pod
在以下部分,我们在 AKS 群集中创建一个 MongoDB Locust 客户端 Pod。 该 Pod 使用之前生成的 Docker 映像来模拟 MongoDB 群集上的工作负荷。 以下规范使用早期教程中创建的外部机密中的密码和用户名。
使用
kubectl apply
命令创建 MongoDB Locust 客户端 Pod。kubectl apply -n mongodb -f - <<EOF apiVersion: apps/v1 kind: Deployment metadata: name: mongodb-locust-client labels: app: mongodb-locust-client spec: replicas: 1 selector: matchLabels: app: mongodb-locust-client template: metadata: labels: app: mongodb-locust-client spec: nodeSelector: kubernetes.azure.com/agentpool: "systempool" # Ensure pods are scheduled on the system node pool containers: - name: mongodb-locust-client image: $MY_ACR_REGISTRY.azurecr.cn/mongodb-locust-client command: ["locust", "-f", "/locustfile.py", "--host=http://${MY_CLUSTER_NAME}-mongodb-mongos.mongodb.svc.cluster.local:27017/"] env: - name: MONGODB_USERNAME valueFrom: secretKeyRef: name: ${AKS_MONGODB_SECRETS_NAME} # Name of your external secret key: MONGODB_DATABASE_ADMIN_USER # Key for the admin username - name: MONGODB_PASSWORD valueFrom: secretKeyRef: name: ${AKS_MONGODB_SECRETS_NAME} # Name of your external secret key: MONGODB_DATABASE_ADMIN_PASSWORD # Key for the admin password EOF
示例输出:
deployment.apps/mongodb-locust-client created
使用
kubectl get pods
命令获取 MongoDB 客户端 Pod 的名称。kubectl get pods -n $AKS_MONGODB_NAMESPACE
示例输出:
NAME READY STATUS RESTARTS AGE ... // Additional pods mongodb-locust-client-564695f8f-q6745 1/1 Running 0 5d8h ... // Additional pods
记下带
mongodb-locust-client
前缀的 Pod 名称。 该 Pod 连接到 MongoDB 群集以执行 MongoDB 命令。
端口转发 MongoDB Locust 客户端 Pod
端口转发端口 8089,以使用
kubectl port-forward
命令访问本地计算机上的 Locust Web 界面。 确保将mongodb-locust-client-564695f8f-q6745
替换为在上一部分创建的 MongoDB 客户端 Pod 的名称。kubectl -n $AKS_MONGODB_NAMESPACE port-forward mongodb-locust-client-564695f8f-q6745 8089:8089
访问 Locust Web 界面
通过在 Web 浏览器中导航到
http://localhost:8089
来访问 Locust Web 界面。 使用 Locust Web 界面,你可以配置负载测试的用户数量和生成率。 还可以实时查看负载测试的统计信息。
连接到 MongoDB shell 并标识主成员
使用以下命令连接到 MongoDB 客户端 Pod。 确保将
mongodb-locust-client-564695f8f-q6745
替换为之前创建的 MongoDB 客户端 Pod 的名称。kubectl -n $AKS_MONGODB_NAMESPACE exec -it mongodb-locust-client-564695f8f-q6745 -- /bin/bash
连接到 Pod 后,使用以下命令连接到 MongoDB shell。 请确认你在
${MY_CLUSTER_NAME}
的步骤 3 中获取的${databaseAdmin}
、${databaseAdminPassword}
和的值是否正确。mongosh --host ${MY_CLUSTER_NAME}-mongodb-rs0-0.${MY_CLUSTER_NAME}-mongodb-rs0.mongodb.svc.cluster.local --port 27017 -u ${databaseAdmin} -p ${databaseAdminPassword} --authenticationDatabase admin
连接到 MongoDB shell 后,使用以下命令来标识主 MongoDB 副本集:
rs.status()
示例输出:
{ set: 'rs0', ... // Additional details members: [ { _id: 0, name: 'cluster-aks-mongodb-rs0-0.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017', health: 1, state: 2, stateStr: 'SECONDARY', uptime: 207037, ... // Additional details for the first member }, { _id: 1, name: 'cluster-aks-mongodb-rs0-1.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017', health: 1, state: 1, stateStr: 'PRIMARY', uptime: 207033, ... // Additional details for the second member }, { _id: 2, name: 'cluster-aks-mongodb-rs0-2.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017', health: 1, state: 2, stateStr: 'SECONDARY', uptime: 207033, ... // Additional details for the third member } ], ... }
从输出中记下状态为“
PRIMARY
”的成员。 在此示例中,带name: cluster-aks-mongodb-rs0-1
的成员是 MongoDB 副本集的主成员。 我们在下一步删除主成员以模拟 MongoDB Pod 故障。
复原能力测试场景
当数据库脚本在第一个终端窗口中继续执行时,我们可以在第二个终端窗口中使用以下脚本模拟故障场景并观察 MongoDB 群集的行为。
场景 1:模拟删除 statefulSet
使用
kubectl get
命令列出所有 StatefulSet。kubectl get sts -n $AKS_MONGODB_NAMESPACE
示例输出:
NAME READY AGE cluster-aks-mongodb-cfg 3/3 5d21h cluster-aks-mongodb-mongos 3/3 5d18h cluster-aks-mongodb-rs0 3/3 3h26m
使用
cluster-aks-mongodb-rs0
命令删除名为kubectl delete
的 StatefulSet。kubectl delete sts -n $AKS_MONGODB_NAMESPACE cluster-aks-mongodb-rs0
删除 StatefulSet 后,检查 Locust 测试仪表板以观察 MongoDB 群集的行为。
使用
kubectl logs
命令获取 Percona Operator Pod 的日志。 确保将<percona-operator-pod>
替换为你的 Percona Operator Pod 的名称。kubectl logs -n $AKS_MONGODB_NAMESPACE <percona-operator-pod>
你应该看到 Percona Operator 重新创建了 StatefulSet 且 MongoDB 群集恢复正常运行,如以下示例输出所示:
... 2024-11-29T13:48:07.427Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "2e037504-d213-4bf7-a2b8-5af396b2d16c", "replset": "rs0", "error": "dial: ping mongo: server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, ... ... 2024-11-29T13:48:37.678Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "2e037504-d213-4bf7-a2b8-5af396b2d16c"} 2024-11-29T13:48:37.678Z ERROR Reconciler error {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "2e037504-d213-4bf7-a2b8-5af396b2d16c", "error": "create pbm object: create PBM connection to cluster-aks-mongodb-rs0-0. cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017,cluster-aks-mongodb-rs0-1.cluster-aks-mongodb-rs0.mongodb.svc.cluster. local:27017,cluster-aks-mongodb-rs0-2.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017: create mongo connection: ping: server selection error: server selection timeout, ... ... 2024-11-29T13:48:38.210Z INFO Waiting for the pods {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "2d970945-86b2-4c10-a610-7f7c42ad9a9f", "replset": "rs0", "size": 3, "pods": 1} 2024-11-29T13:48:38.437Z INFO Cluster state changed {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "2d970945-86b2-4c10-a610-7f7c42ad9a9f", "previous": "error", "current": "initializing"} 2024-11-29T13:48:38.955Z INFO Waiting for the pods {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "f2f803cb-d5f4-4851-9a05-71d46d95ca6d", "replset": "rs0", "size": 3, "pods": 1} 2024-11-29T13:48:40.102Z INFO Waiting for the pods {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "76accdff-7d7a-4ba8-9914-ff8793226dd5", "replset": "rs0", "size": 3, "pods": 1} 2024-11-29T13:48:43.964Z INFO Waiting for the pods {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "c49f50e7-881e-469c-bdf5-aaf925436097", "replset": "rs0", "size": 3, "pods": 1} 2024-11-29T13:48:49.736Z INFO Waiting for the pods {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "5b52ce2e-e07e-43e8-8ca0-b6850a4d8823", "replset": "rs0", "size": 3, "pods": 1} 2024-11-29T13:48:55.560Z INFO Waiting for the pods {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "95a26aff-b022-4543-a22e-0d50ea292190", "replset": "rs0", "size": 3, "pods": 2} 2024-11-29T13:48:56.466Z INFO Waiting for the pods {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "9f64bdb1-e616-44ab-a480-7eaf813db73e", "replset": "rs0", "size": 3, "pods": 2} 2024-11-29T13:49:01.279Z INFO Waiting for the pods {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "0281130b-b13e-4ab2-bd68-0b4cb6d01c84", "replset": "rs0", "size": 3, "pods": 2} 2024-11-29T13:49:07.027Z INFO Waiting for the pods {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "a3caedd1-ead3-406b-8ce3-056c7cd875dc", "replset": "rs0", "size": 3, "pods": 2} 2024-11-29T13:49:12.783Z INFO Waiting for the pods {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "8a82eb0c-8b26-4542-a4b5-f7e13e030d1e", "replset": "rs0", "size": 3, "pods": 2} 2024-11-29T13:49:28.521Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "22fc8c4d-30d5-4bf6-b405-720533f1d8d2", "replset": "rs0", "error": "dial: ping mongo: server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, ... ... 2024-11-29T13:49:32.472Z INFO Fixing member tags {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "8ff97dc9-b024-4cd9-b411-c3ffaf225e14", "replset": "rs0"} 2024-11-29T13:49:39.065Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "28438ddb-77a1-48b0-8390-277c4687a97b", "replset": "rs0", "error": "create system users: failed to get mongo client: ping mongo: connection() error occurred during connection handshake: dial tcp: lookup cluster-aks-mongodb-rs0-1.cluster-aks-mongodb-rs0.mongodb. svc.cluster.local on 10.0.0.10:53: no such host", ... ... 2024-11-29T13:49:40.424Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "cfec8e33-d0c9-4adf-afe9-bc6dcbf711ba", "replset": "rs0", "error": "dial: ping mongo: connection() error occurred during connection handshake: ... ... 2024-11-29T13:49:45.433Z INFO Cluster state changed {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "0765ea5f-52d8-44e2-a680-c7d082dd88e1", "previous": "initializing", "current": "ready"}
场景 2:模拟 MongoDB Pod 故障
标识主 MongoDB 副本集,然后使用
kubectl delete
命令将其删除,以模拟 MongoDB Pod 故障。 确保将cluster-aks-mongodb-rs0-1
替换为rs.status()
命令的输出中的主成员名称。kubectl delete pod cluster-aks-mongodb-rs0-1 -n $AKS_MONGODB_NAMESPACE
MongoDB 副本集应该选择一个新的主成员来继续处理请求。 我们可以通过连接到 MongoDB shell 并执行
rs.status()
命令来验证新的主成员。 使用以下命令连接到 MongoDB 客户端 Pod。 确保将mongodb-locust-client-564695f8f-q6745
替换为上一命令的输出中的 Pod 名称。kubectl -n $AKS_MONGODB_NAMESPACE exec -it mongodb-locust-client-564695f8f-q6745 -- /bin/bash
连接到 Pod 后,使用以下命令连接到 MongoDB shell。 请确认你在
${MY_CLUSTER_NAME}
的步骤 3 中获取的${databaseAdmin}
、${databaseAdminPassword}
和的值是否正确。mongosh --host ${MY_CLUSTER_NAME}-mongodb-rs0-0.${MY_CLUSTER_NAME}-mongodb-rs0.mongodb.svc.cluster.local --port 27017 -u ${databaseAdmin} -p ${databaseAdminPassword} --authenticationDatabase admin
连接到 MongoDB shell 后,使用以下命令来标识主 MongoDB 副本集:
rs.status()
示例输出:
{ set: 'rs0', ... // Additional details members: [ { _id: 0, name: 'cluster-aks-mongodb-rs0-0.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017', health: 1, state: 2, stateStr: 'PRIMARY', uptime: 207037, ... // Additional details for the first member }, { _id: 1, name: 'cluster-aks-mongodb-rs0-1.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017', health: 1, state: 1, stateStr: 'SECONDARY', uptime: 207033, ... // Additional details for the second member }, { _id: 2, name: 'cluster-aks-mongodb-rs0-2.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017', health: 1, state: 2, stateStr: 'SECONDARY', uptime: 207033, ... // Additional details for the third member } ], ... }
你应该看到状态为“
PRIMARY
”的成员更改为“SECONDARY
”,而状态为“SECONDARY
”的另一个成员更改为“PRIMARY
”:使用
kubectl logs
命令获取 Percona Operator Pod 的日志。 确保将<percona-operator-pod>
替换为你的 Percona Operator Pod 的名称。kubectl logs -n $AKS_MONGODB_NAMESPACE <percona-operator-pod>
你应该看到 Percona Operator 重新创建了 StatefulSet 且 MongoDB 群集恢复正常运行,如以下示例输出所示:
... 2024-11-29T14:02:17.151Z INFO StatefulSet is not up to date {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "b1b1b1b1-cccc-dddd-eeee-f2f2f2f2f2f2", "sts": "cluster-aks-mongodb-rs0"} 2024-11-29T14:02:17.189Z INFO StatefulSet is not up to date {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "b1b1b1b1-cccc-dddd-eeee-f2f2f2f2f2f2", "sts": "cluster-aks-mongodb-rs0"} 2024-11-29T14:02:17.405Z INFO Cluster state changed {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "b1b1b1b1-cccc-dddd-eeee-f2f2f2f2f2f2", "previous": "ready", "current": "initializing"} 2024-11-29T14:02:18.448Z INFO StatefulSet is not up to date {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "c2c2c2c2-dddd-eeee-ffff-a3a3a3a3a3a3", "sts": "cluster-aks-mongodb-rs0"} 2024-11-29T14:02:18.489Z INFO StatefulSet is not up to date {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "c2c2c2c2-dddd-eeee-ffff-a3a3a3a3a3a3", "sts": "cluster-aks-mongodb-rs0"} 2024-11-29T14:02:23.359Z INFO StatefulSet is not up to date {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "d3d3d3d3-eeee-ffff-aaaa-b4b4b4b4b4b4", "sts": "cluster-aks-mongodb-rs0"} 2024-11-29T14:02:23.402Z INFO StatefulSet is not up to date {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "d3d3d3d3-eeee-ffff-aaaa-b4b4b4b4b4b4", "sts": "cluster-aks-mongodb-rs0"} 2024-11-29T14:02:29.341Z INFO StatefulSet is not up to date {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "e4e4e4e4-ffff-aaaa-bbbb-c5c5c5c5c5c5", "sts": "cluster-aks-mongodb-rs0"} 2024-11-29T14:02:29.377Z INFO StatefulSet is not up to date {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "e4e4e4e4-ffff-aaaa-bbbb-c5c5c5c5c5c5", "sts": "cluster-aks-mongodb-rs0"} 2024-11-29T14:03:01.246Z INFO Cluster state changed {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "f5f5f5f5-aaaa-bbbb-cccc-d6d6d6d6d6d6", "previous": "initializing", "current": "ready"}