在 Azure Kubernetes 服务上对 MongoDB 群集进行负载测试 (AKS)

使用 Percona Operator for MongoDB 部署 AKS 上的 MongoDB 群集并在群集上安装用来与 MongoDB 交互的 mongo-express 后,可以在多种场景中测试 MongoDB 群集的复原能力。 本教程基于以前的指南中介绍的 MongoDB 群集部署和 MongoDB 群集验证。

生成并运行 MongoDB 的示例客户端应用程序

以下步骤演示如何生成 MongoDB 的示例客户端应用程序,并将应用程序的 Docker 映像推送到 Azure 容器注册表。 示例客户端应用程序使用 Locust 负载测试框架来模拟 MongoDB 群集上的工作负荷。 此应用程序将文档插入 MongoDB 群集并度量每个文档插入操作所花的时间。 此应用程序使用 Faker 库为员工文档生成随机姓名和部门。 可以扩展此类以实现更复杂的操作。

  1. 使用以下命令创建新目录:

    mkdir mongodb-locust-client
    cd mongodb-locust-client
    
  2. 使用以下命令在 requirements.txt 目录中创建 mongodb-locust-client 文件:

    cat > requirements.txt <<EOF
    locust
    pymongo
    Faker
    EOF
    
  3. 使用以下命令在 locustfile.py 目录中创建 mongodb-locust-client

    cat > locustfile.py <<EOF
    import os
    import random
    import logging
    import time
    import uuid
    from locust import User, task, constant_throughput, events, tag, between
    from pymongo import MongoClient
    from pymongo.errors import ServerSelectionTimeoutError
    from pymongo.errors import ConnectionFailure
    from faker import Faker
    
    # Set up Faker for generating random names and departments
    fake = Faker()
    
    # Configure logging
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger(__name__)
    
    # List of department names for random selection
    department_keywords = [
        "Accounting", "Marketing", "Human Resources", "Sales", "Research and Development",
        "IT", "Customer Support", "Legal", "Operations", "Finance", "Product Management",
        "Public Relations", "Engineering", "Data Science", "Business Development"
    ]
    
    class MongoDBLocust(User):
        # Set constant throughput (requests per second)
        wait_time = constant_throughput(20)
        # MongoDB connection details (make sure these are set correctly in your environment)
        host = "${MY_CLUSTER_NAME}-mongodb-mongos.mongodb.svc.cluster.local"
        port = "27017"
        username = os.getenv('MONGODB_USERNAME')
        password = os.getenv('MONGODB_PASSWORD')
    
        def __init__(self, *args, **kwargs):
            super(MongoDBLocust, self).__init__(*args, **kwargs)
            self.client = MongoDBClient(self.username, self.password)  # Initialize MongoDB client
    
        def on_stop(self):
            """Called when the simulated user stops, to close the MongoDB client."""
            self.client.close()
    
        @task
        @tag("insert")
        def insert_document(self):
            """Task to insert a document into MongoDB."""
            self.client.insert_document("insert_document")
    
    class MongoDBClient(object):
        """MongoDB client for managing database interactions."""
    
        def __init__(self, username, password):
            # Set MongoDB connection parameters
            self.host = "${MY_CLUSTER_NAME}-mongodb-mongos.mongodb.svc.cluster.local"
            self.port = "27017"
            self.username = username
            self.password = password
            self.client = MongoClient(f"mongodb://{self.username}:{self.password}@{self.host}:{self.port}/")
            self.db = self.client['employeeDB']
            self.collection = self.db['employees']
            self.document_id = None  # Placeholder for the inserted document ID
    
        def insert_document(self, task_name, command='INSERT'):
            """Insert a document into MongoDB and measure the time."""
            employee_id = str(uuid.uuid4())
            start_time = time.time()
            logger.info(f"{employee_id} - Start Time -> start_time: {start_time}")
            try:
                # Create a new fake employee document
                new_document = {
                    'Id': str(uuid.uuid4()),
                    'EmployeeId': employee_id,
                    'FirstName': fake.first_name(),
                    'LastName': fake.last_name(),
                    'Department': random.choice(department_keywords)
                }
                # Insert the document
                result = self.collection.insert_one(new_document)
                self.document_id = str(result.inserted_id)  # Store the document ID for future reads
    
                # Measure the time taken for the MongoDB operation (in milliseconds)
                total_time = (time.time() - start_time) * 1000  # Convert to ms
                logger.info(f"{employee_id} - Task - {task_name} -> total_time: {total_time} ms")
    
                # Fire the event to report the request
                events.request.fire(
                    request_type=command,
                    name=task_name,
                    response_time=total_time,
                    response_length=len(str(result))  # Length of the result returned by the operation
                )
            except Exception as e:
                total_time = (time.time() - start_time) * 1000  # Time taken with error
                logger.error(f"{employee_id} - Error in {task_name}: {e} -> total_time: {total_time} ms")
                events.request.fire(
                    request_type=command,
                    name=task_name,
                    response_time=total_time,
                    response_length=0,  # No response in case of error
                    exception=e
                )
            logger.info(f"{employee_id} - End Time -> end time: {time.time()}")
        def close(self):
            """Close the MongoDB connection."""
            self.client.close()
    
    EOF
    
  4. 使用以下命令在 mongodb-locust-client 目录中创建 Dockerfile。 Dockerfile 为示例客户端应用程序创建一个 Docker 映像,使用 python:3.10-slim-bullseye 映像作为基础映像,并为应用程序安装所需的依赖项。

    cat > Dockerfile <<EOF
    # Use a slim Python image as the base
    FROM python:3.10-slim-bullseye
    
    # Install wget and gnupg
    RUN apt-get update && apt-get install -y wget gnupg
    
    # Add MongoDB GPG Key and repository
    RUN wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | tee /etc/apt/trusted.gpg.d/mongodb.asc && \
        echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/debian bullseye/mongodb-org/6.0 main" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list
    
    # Update APT sources and install MongoDB Shell (mongosh)
    RUN apt-get update && apt-get install -y mongodb-mongosh
    
    # Copy your application files
    COPY requirements.txt ./
    COPY locustfile.py ./
    
    # Install Python dependencies
    RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt
    
    EOF
    
  5. 使用 az acr build 命令为示例客户端应用程序生成 Docker 映像。 确保将 $MY_ACR_REGISTRY 替换为 Azure 容器注册表的名称。

    az acr build --image mongodb-locust-client --registry $MY_ACR_REGISTRY .
    

测试 AKS 上的 MongoDB 群集

创建 MongoDB Locust 客户端 Pod

在以下部分,我们在 AKS 群集中创建一个 MongoDB Locust 客户端 Pod。 该 Pod 使用之前生成的 Docker 映像来模拟 MongoDB 群集上的工作负荷。 以下规范使用早期教程中创建的外部机密中的密码和用户名。

  1. 使用 kubectl apply 命令创建 MongoDB Locust 客户端 Pod。

    kubectl apply -n mongodb -f - <<EOF
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: mongodb-locust-client
      labels:
        app: mongodb-locust-client
    spec:
      replicas: 1
      selector:
        matchLabels:
          app: mongodb-locust-client
      template:
        metadata:
          labels:
            app: mongodb-locust-client
        spec:
          nodeSelector:
            kubernetes.azure.com/agentpool: "systempool"  # Ensure pods are scheduled on the system node pool
          containers:
            - name: mongodb-locust-client
              image: $MY_ACR_REGISTRY.azurecr.cn/mongodb-locust-client
              command: ["locust", "-f", "/locustfile.py", "--host=http://${MY_CLUSTER_NAME}-mongodb-mongos.mongodb.svc.cluster.local:27017/"]
              env:
                - name: MONGODB_USERNAME
                  valueFrom:
                    secretKeyRef:
                      name: ${AKS_MONGODB_SECRETS_NAME}  # Name of your external secret
                      key: MONGODB_DATABASE_ADMIN_USER  # Key for the admin username
                - name: MONGODB_PASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: ${AKS_MONGODB_SECRETS_NAME}  # Name of your external secret
                      key: MONGODB_DATABASE_ADMIN_PASSWORD  # Key for the admin password
    EOF
    

    示例输出:

    deployment.apps/mongodb-locust-client created
    
  2. 使用 kubectl get pods 命令获取 MongoDB 客户端 Pod 的名称。

    kubectl get pods -n $AKS_MONGODB_NAMESPACE
    

    示例输出:

    NAME                                                READY   STATUS    RESTARTS   AGE
    ... // Additional pods
    mongodb-locust-client-564695f8f-q6745               1/1     Running   0          5d8h
    ... // Additional pods
    
  3. 记下带 mongodb-locust-client 前缀的 Pod 名称。 该 Pod 连接到 MongoDB 群集以执行 MongoDB 命令。

端口转发 MongoDB Locust 客户端 Pod

  • 端口转发端口 8089,以使用 kubectl port-forward 命令访问本地计算机上的 Locust Web 界面。 确保将 mongodb-locust-client-564695f8f-q6745 替换为在上一部分创建的 MongoDB 客户端 Pod 的名称。

    kubectl -n $AKS_MONGODB_NAMESPACE port-forward mongodb-locust-client-564695f8f-q6745 8089:8089
    

访问 Locust Web 界面

  • 通过在 Web 浏览器中导航到 http://localhost:8089 来访问 Locust Web 界面。 使用 Locust Web 界面,你可以配置负载测试的用户数量和生成率。 还可以实时查看负载测试的统计信息。

    显示 Locust 测试仪表板的网页的屏幕截图。

连接到 MongoDB shell 并标识主成员

  1. 使用以下命令连接到 MongoDB 客户端 Pod。 确保将 mongodb-locust-client-564695f8f-q6745 替换为之前创建的 MongoDB 客户端 Pod 的名称。

    kubectl -n $AKS_MONGODB_NAMESPACE exec -it mongodb-locust-client-564695f8f-q6745 -- /bin/bash
    
  2. 连接到 Pod 后,使用以下命令连接到 MongoDB shell。 请确认你在${MY_CLUSTER_NAME}的步骤 3 中获取的${databaseAdmin}${databaseAdminPassword}的值是否正确。

    mongosh --host ${MY_CLUSTER_NAME}-mongodb-rs0-0.${MY_CLUSTER_NAME}-mongodb-rs0.mongodb.svc.cluster.local --port 27017 -u ${databaseAdmin} -p ${databaseAdminPassword} --authenticationDatabase admin
    
  3. 连接到 MongoDB shell 后,使用以下命令来标识主 MongoDB 副本集:

    rs.status()
    

    示例输出:

    {
      set: 'rs0',
      ... // Additional details
      members: [
        {
          _id: 0,
          name: 'cluster-aks-mongodb-rs0-0.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017',
          health: 1,
          state: 2,
          stateStr: 'SECONDARY',
          uptime: 207037,
          ... // Additional details for the first member
        },
        {
          _id: 1,
          name: 'cluster-aks-mongodb-rs0-1.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017',
          health: 1,
          state: 1,
          stateStr: 'PRIMARY',
          uptime: 207033,
          ... // Additional details for the second member
        },
        {
          _id: 2,
          name: 'cluster-aks-mongodb-rs0-2.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017',
          health: 1,
          state: 2,
          stateStr: 'SECONDARY',
          uptime: 207033,
          ... // Additional details for the third member
        }
      ],
      ...
    }
    
  4. 从输出中记下状态为“PRIMARY”的成员。 在此示例中,带 name: cluster-aks-mongodb-rs0-1 的成员是 MongoDB 副本集的主成员。 我们在下一步删除主成员以模拟 MongoDB Pod 故障。

复原能力测试场景

当数据库脚本在第一个终端窗口中继续执行时,我们可以在第二个终端窗口中使用以下脚本模拟故障场景并观察 MongoDB 群集的行为。

场景 1:模拟删除 statefulSet

  1. 使用 kubectl get 命令列出所有 StatefulSet。

    kubectl get sts -n $AKS_MONGODB_NAMESPACE
    

    示例输出:

    NAME                         READY   AGE
    cluster-aks-mongodb-cfg      3/3     5d21h
    cluster-aks-mongodb-mongos   3/3     5d18h
    cluster-aks-mongodb-rs0      3/3     3h26m
    
  2. 使用 cluster-aks-mongodb-rs0 命令删除名为 kubectl delete 的 StatefulSet。

    kubectl delete sts -n $AKS_MONGODB_NAMESPACE cluster-aks-mongodb-rs0
    
  3. 删除 StatefulSet 后,检查 Locust 测试仪表板以观察 MongoDB 群集的行为。

    一个网页的屏幕截图,显示删除 StatefulSet 时的 Locust 测试仪表板。

  4. 使用 kubectl logs 命令获取 Percona Operator Pod 的日志。 确保将 <percona-operator-pod> 替换为你的 Percona Operator Pod 的名称。

    kubectl logs -n $AKS_MONGODB_NAMESPACE <percona-operator-pod>
    

    你应该看到 Percona Operator 重新创建了 StatefulSet 且 MongoDB 群集恢复正常运行,如以下示例输出所示:

    ...
    2024-11-29T13:48:07.427Z        ERROR   failed to reconcile cluster     {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "2e037504-d213-4bf7-a2b8-5af396b2d16c", "replset": "rs0", "error": "dial: ping mongo: server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, ...
    ...
    2024-11-29T13:48:37.678Z        INFO    Warning: Reconciler returned both a non-zero result and a non-nil error. The result will  always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see:  https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler   {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "2e037504-d213-4bf7-a2b8-5af396b2d16c"}
    2024-11-29T13:48:37.678Z        ERROR   Reconciler error        {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "2e037504-d213-4bf7-a2b8-5af396b2d16c", "error": "create pbm object: create PBM connection to cluster-aks-mongodb-rs0-0. cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017,cluster-aks-mongodb-rs0-1.cluster-aks-mongodb-rs0.mongodb.svc.cluster. local:27017,cluster-aks-mongodb-rs0-2.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017: create mongo connection: ping: server  selection error: server selection timeout, ...
    ...
    2024-11-29T13:48:38.210Z        INFO    Waiting for the pods    {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "2d970945-86b2-4c10-a610-7f7c42ad9a9f", "replset": "rs0", "size": 3, "pods": 1}
    2024-11-29T13:48:38.437Z        INFO    Cluster state changed   {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "2d970945-86b2-4c10-a610-7f7c42ad9a9f", "previous": "error", "current": "initializing"}
    2024-11-29T13:48:38.955Z        INFO    Waiting for the pods    {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "f2f803cb-d5f4-4851-9a05-71d46d95ca6d", "replset": "rs0", "size": 3, "pods": 1}
    2024-11-29T13:48:40.102Z        INFO    Waiting for the pods    {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "76accdff-7d7a-4ba8-9914-ff8793226dd5", "replset": "rs0", "size": 3, "pods": 1}
    2024-11-29T13:48:43.964Z        INFO    Waiting for the pods    {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "c49f50e7-881e-469c-bdf5-aaf925436097", "replset": "rs0", "size": 3, "pods": 1}
    2024-11-29T13:48:49.736Z        INFO    Waiting for the pods    {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "5b52ce2e-e07e-43e8-8ca0-b6850a4d8823", "replset": "rs0", "size": 3, "pods": 1}
    2024-11-29T13:48:55.560Z        INFO    Waiting for the pods    {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "95a26aff-b022-4543-a22e-0d50ea292190", "replset": "rs0", "size": 3, "pods": 2}
    2024-11-29T13:48:56.466Z        INFO    Waiting for the pods    {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "9f64bdb1-e616-44ab-a480-7eaf813db73e", "replset": "rs0", "size": 3, "pods": 2}
    2024-11-29T13:49:01.279Z        INFO    Waiting for the pods    {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "0281130b-b13e-4ab2-bd68-0b4cb6d01c84", "replset": "rs0", "size": 3, "pods": 2}
    2024-11-29T13:49:07.027Z        INFO    Waiting for the pods    {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "a3caedd1-ead3-406b-8ce3-056c7cd875dc", "replset": "rs0", "size": 3, "pods": 2}
    2024-11-29T13:49:12.783Z        INFO    Waiting for the pods    {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "8a82eb0c-8b26-4542-a4b5-f7e13e030d1e", "replset": "rs0", "size": 3, "pods": 2}
    2024-11-29T13:49:28.521Z        ERROR   failed to reconcile cluster     {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "22fc8c4d-30d5-4bf6-b405-720533f1d8d2", "replset": "rs0", "error": "dial: ping mongo: server selection error: context deadline  exceeded, current topology: { Type: ReplicaSetNoPrimary, ...
    ...
    2024-11-29T13:49:32.472Z        INFO    Fixing member tags      {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "8ff97dc9-b024-4cd9-b411-c3ffaf225e14", "replset": "rs0"}
    2024-11-29T13:49:39.065Z        ERROR   failed to reconcile cluster     {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "28438ddb-77a1-48b0-8390-277c4687a97b", "replset": "rs0", "error": "create system users: failed to get mongo client: ping mongo:  connection() error occurred during connection handshake: dial tcp: lookup cluster-aks-mongodb-rs0-1.cluster-aks-mongodb-rs0.mongodb. svc.cluster.local on 10.0.0.10:53: no such host", ...
    ...
    2024-11-29T13:49:40.424Z        ERROR   failed to reconcile cluster     {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "cfec8e33-d0c9-4adf-afe9-bc6dcbf711ba", "replset": "rs0", "error": "dial: ping mongo: connection() error occurred during connection  handshake: ...
    ...
    2024-11-29T13:49:45.433Z        INFO    Cluster state changed   {"controller": "psmdb-controller", "object":  {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":  "0765ea5f-52d8-44e2-a680-c7d082dd88e1", "previous": "initializing", "current": "ready"}
    

场景 2:模拟 MongoDB Pod 故障

  1. 标识主 MongoDB 副本集,然后使用 kubectl delete 命令将其删除,以模拟 MongoDB Pod 故障。 确保将 cluster-aks-mongodb-rs0-1 替换为 rs.status() 命令的输出中的主成员名称。

    kubectl delete pod cluster-aks-mongodb-rs0-1 -n $AKS_MONGODB_NAMESPACE
    
  2. MongoDB 副本集应该选择一个新的主成员来继续处理请求。 我们可以通过连接到 MongoDB shell 并执行 rs.status() 命令来验证新的主成员。 使用以下命令连接到 MongoDB 客户端 Pod。 确保将 mongodb-locust-client-564695f8f-q6745 替换为上一命令的输出中的 Pod 名称。

    kubectl -n $AKS_MONGODB_NAMESPACE exec -it mongodb-locust-client-564695f8f-q6745 -- /bin/bash
    
  3. 连接到 Pod 后,使用以下命令连接到 MongoDB shell。 请确认你在${MY_CLUSTER_NAME}的步骤 3 中获取的${databaseAdmin}${databaseAdminPassword}的值是否正确。

    mongosh --host ${MY_CLUSTER_NAME}-mongodb-rs0-0.${MY_CLUSTER_NAME}-mongodb-rs0.mongodb.svc.cluster.local --port 27017 -u ${databaseAdmin} -p ${databaseAdminPassword} --authenticationDatabase admin
    
  4. 连接到 MongoDB shell 后,使用以下命令来标识主 MongoDB 副本集:

    rs.status()
    

    示例输出:

    {
      set: 'rs0',
      ... // Additional details
      members: [
        {
          _id: 0,
          name: 'cluster-aks-mongodb-rs0-0.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017',
          health: 1,
          state: 2,
          stateStr: 'PRIMARY',
          uptime: 207037,
          ... // Additional details for the first member
        },
        {
          _id: 1,
          name: 'cluster-aks-mongodb-rs0-1.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017',
          health: 1,
          state: 1,
          stateStr: 'SECONDARY',
          uptime: 207033,
          ... // Additional details for the second member
        },
        {
          _id: 2,
          name: 'cluster-aks-mongodb-rs0-2.cluster-aks-mongodb-rs0.mongodb.svc.cluster.local:27017',
          health: 1,
          state: 2,
          stateStr: 'SECONDARY',
          uptime: 207033,
          ... // Additional details for the third member
        }
      ],
      ...
    }
    

    你应该看到状态为“PRIMARY”的成员更改为“SECONDARY”,而状态为“SECONDARY”的另一个成员更改为“PRIMARY”:

    一个网页的屏幕截图,显示删除 Pod 时的 Locust 测试仪表板。

  5. 使用 kubectl logs 命令获取 Percona Operator Pod 的日志。 确保将 <percona-operator-pod> 替换为你的 Percona Operator Pod 的名称。

    kubectl logs -n $AKS_MONGODB_NAMESPACE <percona-operator-pod>
    

    你应该看到 Percona Operator 重新创建了 StatefulSet 且 MongoDB 群集恢复正常运行,如以下示例输出所示:

    ...
    2024-11-29T14:02:17.151Z        INFO    StatefulSet is not up to date   {"controller": "psmdb-controller", "object": {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID": "b1b1b1b1-cccc-dddd-eeee-f2f2f2f2f2f2", "sts": "cluster-aks-mongodb-rs0"}
    2024-11-29T14:02:17.189Z        INFO    StatefulSet is not up to date   {"controller": "psmdb-controller", "object":   {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":   "b1b1b1b1-cccc-dddd-eeee-f2f2f2f2f2f2", "sts": "cluster-aks-mongodb-rs0"}
    2024-11-29T14:02:17.405Z        INFO    Cluster state changed   {"controller": "psmdb-controller", "object":   {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":   "b1b1b1b1-cccc-dddd-eeee-f2f2f2f2f2f2", "previous": "ready", "current": "initializing"}
    2024-11-29T14:02:18.448Z        INFO    StatefulSet is not up to date   {"controller": "psmdb-controller", "object":   {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":   "c2c2c2c2-dddd-eeee-ffff-a3a3a3a3a3a3", "sts": "cluster-aks-mongodb-rs0"}
    2024-11-29T14:02:18.489Z        INFO    StatefulSet is not up to date   {"controller": "psmdb-controller", "object":   {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":   "c2c2c2c2-dddd-eeee-ffff-a3a3a3a3a3a3", "sts": "cluster-aks-mongodb-rs0"}
    2024-11-29T14:02:23.359Z        INFO    StatefulSet is not up to date   {"controller": "psmdb-controller", "object":   {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":   "d3d3d3d3-eeee-ffff-aaaa-b4b4b4b4b4b4", "sts": "cluster-aks-mongodb-rs0"}
    2024-11-29T14:02:23.402Z        INFO    StatefulSet is not up to date   {"controller": "psmdb-controller", "object":   {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":   "d3d3d3d3-eeee-ffff-aaaa-b4b4b4b4b4b4", "sts": "cluster-aks-mongodb-rs0"}
    2024-11-29T14:02:29.341Z        INFO    StatefulSet is not up to date   {"controller": "psmdb-controller", "object":   {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":   "e4e4e4e4-ffff-aaaa-bbbb-c5c5c5c5c5c5", "sts": "cluster-aks-mongodb-rs0"}
    2024-11-29T14:02:29.377Z        INFO    StatefulSet is not up to date   {"controller": "psmdb-controller", "object":   {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":   "e4e4e4e4-ffff-aaaa-bbbb-c5c5c5c5c5c5", "sts": "cluster-aks-mongodb-rs0"}
    2024-11-29T14:03:01.246Z        INFO    Cluster state changed   {"controller": "psmdb-controller", "object":   {"name":"cluster-aks-mongodb","namespace":"mongodb"}, "namespace": "mongodb", "name": "cluster-aks-mongodb", "reconcileID":   "f5f5f5f5-aaaa-bbbb-cccc-d6d6d6d6d6d6", "previous": "initializing", "current": "ready"}
    

后续步骤