1

I have a Mongo deployment with a metrics exporter sidecar. Now when I load the associated dashboard in Grafana the metrics are not showing - it appears the exporter is not scrapping all metrics :

dash

What is working

Only the Mongo UP metric is working i. e mongodb_up{env=~""} and part of Server Metrics

What is not working

All of the following metrics in the dashboard are showing no data : opscounters, Replication Set Metrics, Cursor Metrics

My configuration: Deployment.yaml (using Percona MongoDB Exporter)

apiVersion: apps/v1
kind: Deployment
metadata:
  name: mongodb-prom
  namespace: "labs"
  labels:
    app.kubernetes.io/name: mongodb
    helm.sh/chart: mongodb-12.1.12
    app.kubernetes.io/instance: mongodb-prom
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: mongodb
spec:
  replicas: 1
  strategy:
    type: RollingUpdate
  selector:
    matchLabels:
      app.kubernetes.io/name: mongodb
      app.kubernetes.io/instance: mongodb-prom
      app.kubernetes.io/component: mongodb
  template:
    metadata:
      labels:
        app.kubernetes.io/name: mongodb
        helm.sh/chart: mongodb-12.1.12
        app.kubernetes.io/instance: mongodb-prom
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: mongodb
    spec:
  
  serviceAccountName: mongodb-prom
  affinity:
      
    podAntiAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
        - podAffinityTerm:
            labelSelector:
              matchLabels:
                app.kubernetes.io/name: mongodb
                app.kubernetes.io/instance: mongodb-prom
                app.kubernetes.io/component: mongodb
            namespaces:
              - "labs"
            topologyKey: kubernetes.io/hostname
          weight: 100
      
  securityContext:
    fsGroup: 1001
    sysctls: []
  
  containers:
    - name: mongodb
      image: docker.io/bitnami/mongodb:5.0.9-debian-10-r15
      imagePullPolicy: "IfNotPresent"
      securityContext:
        runAsNonRoot: true
        runAsUser: 1001
      env:
        - name: BITNAMI_DEBUG
          value: "false"
        - name: MONGODB_ROOT_USER
          value: "root"
        - name: MONGODB_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: mongodb-prom
              key: mongodb-root-password
        - name: ALLOW_EMPTY_PASSWORD
          value: "no"
        - name: MONGODB_SYSTEM_LOG_VERBOSITY
          value: "0"
        - name: MONGODB_DISABLE_SYSTEM_LOG
          value: "no"
        - name: MONGODB_DISABLE_JAVASCRIPT
          value: "no"
        - name: MONGODB_ENABLE_JOURNAL
          value: "yes"
        - name: MONGODB_PORT_NUMBER
          value: "27017"
        - name: MONGODB_ENABLE_IPV6
          value: "no"
        - name: MONGODB_ENABLE_DIRECTORY_PER_DB
          value: "no"
      ports:
        - name: mongodb
          containerPort: 27017
      
      volumeMounts:
        - name: datadir
          mountPath: /bitnami/mongodb
        - name: datadir
          mountPath: /tmp           
    - name: metrics
      image: percona/mongodb_exporter:0.35
      imagePullPolicy: "IfNotPresent"
      args:
      - "--mongodb.direct-connect=false"
      - "--mongodb.uri=mongodb://username:password@mongodb-prom/admin"
      ports:
        - name: metrics
          containerPort: 9216
      resources:
        requests:
          memory: 128Mi
          cpu: 250m
  volumes:
    - name: datadir
      persistentVolumeClaim:
        claimName: mongodb

metrics-svc.yaml

apiVersion: v1
kind: Service
metadata:
  name: mongodb-metrics
  namespace: "labs"
  labels:
    app.kubernetes.io/name: mongodb
    helm.sh/chart: mongodb-12.1.12
    app.kubernetes.io/instance: mongodb-prom
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
  annotations:
    prometheus.io/path: /metrics
    prometheus.io/port: '9216'
    prometheus.io/scrape: "true"
spec:
  type: ClusterIP
  ports:
    - port: 9216
      targetPort: metrics
      protocol: TCP
      name: http-metrics
  selector:
    app.kubernetes.io/name: mongodb
    app.kubernetes.io/instance: mongodb-prom
    app.kubernetes.io/component: mongodb

service.yaml

apiVersion: v1
kind: Service
metadata:
  name: mongodb-prom
  namespace: "labs"
  labels:
    app.kubernetes.io/name: mongodb
    helm.sh/chart: mongodb-12.1.12
    app.kubernetes.io/instance: mongodb-prom
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: mongodb
spec:
  type: ClusterIP
  sessionAffinity: None
  ports:
    - name: "mongodb"
      port: 27017
      targetPort: mongodb
  selector:
    app.kubernetes.io/name: mongodb
    app.kubernetes.io/instance: mongodb-prom
    app.kubernetes.io/component: mongodb

What I have tried

  1. I have tried using the Bitnami MongoDB Exporter version for the sidecar. It yields the exact same results :

    - name: metrics
          image: docker.io/bitnami/mongodb-exporter:0.32.0-debian-11-r5
          imagePullPolicy: "IfNotPresent"
          securityContext:
            runAsNonRoot: true
            runAsUser: 1001
          command:
            - /bin/bash
            - -ec
          args:
            - |
              /bin/mongodb_exporter --web.listen-address ":9216" --mongodb.uri "mongodb://$MONGODB_ROOT_USER:$(echo $MONGODB_ROOT_PASSWORD | sed -r "s/@/%40/g;s/:/%3A/g")@localhost:27017/admin?" 
          env:
            - name: MONGODB_ROOT_USER
              value: "root"
            - name: MONGODB_ROOT_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: mongodb-prom
                  key: mongodb-root-password
          ports:
            - name: metrics
              containerPort: 9216
          livenessProbe:
            failureThreshold: 3
            initialDelaySeconds: 15
            periodSeconds: 5
            successThreshold: 1
            timeoutSeconds: 5
            httpGet:
              path: /
              port: metrics
          readinessProbe:
            failureThreshold: 3
            initialDelaySeconds: 5
            periodSeconds: 5
            successThreshold: 1
            timeoutSeconds: 1
            httpGet:
              path: /
              port: metrics
          resources:
            limits: {}
            requests: {}
            ```
    
    
  2. I am following the reference implementation given here and I have added the clusterMonitor role to the db user as below:

    db.grantRolesToUser("root",[{ role: "clusterMonitor", db: "admin" }, { role: "read", db: "local" }])

and when I execute db.getUsers(); I get

    [
        {
                "_id" : "admin.root",
                "userId" : UUID("d8e181fc-6429-447e-bbcb-cec252f0792f"),
                "user" : "root",
                "db" : "admin",
                "roles" : [
                        {
                                "role" : "clusterMonitor",
                                "db" : "admin"
                        },
                        {
                                "role" : "root",
                                "db" : "admin"
                        },
                        {
                                "role" : "read",
                                "db" : "local"
                        }
                ],
                "mechanisms" : [
                        "SCRAM-SHA-1",
                        "SCRAM-SHA-256"
                ]
        }
]

Even after granting these roles the dashboard is still not loading the missing metrics.

  1. I have updated the Prometheus and Grafana versions in the dashboard's JSON to match my installed versions (Could this affect anything?)

The default dashboard I am using is here.

What am I missing ?

1 Answer 1

1

For anyone struggling with this I had to add the --collect-all and --compatibility-mode flags for the container to pull the metrics as in the configuration below:

apiVersion: apps/v1
kind: Deployment
metadata:
  name: mongodb-prom
  namespace: "labs"
  labels:
    app.kubernetes.io/name: mongodb
    helm.sh/chart: mongodb-12.1.12
    app.kubernetes.io/instance: mongodb-prom
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: mongodb
spec:
  replicas: 1
  strategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate  
  selector:
    matchLabels:
      app.kubernetes.io/name: mongodb
      app.kubernetes.io/instance: mongodb-prom
      app.kubernetes.io/component: mongodb
  template:
    metadata:
      labels:
        app.kubernetes.io/name: mongodb
        helm.sh/chart: mongodb-12.1.12
        app.kubernetes.io/instance: mongodb-prom
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: mongodb
    spec:
  
  serviceAccountName: mongodb-prom
  affinity:
      
    podAntiAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
        - podAffinityTerm:
            labelSelector:
              matchLabels:
                app.kubernetes.io/name: mongodb
                app.kubernetes.io/instance: mongodb-prom
                app.kubernetes.io/component: mongodb
            namespaces:
              - "labs"
            topologyKey: kubernetes.io/hostname
          weight: 100
      
  securityContext:
    fsGroup: 1001
    sysctls: []
  
  containers:
    - name: mongodb
      image: docker.io/bitnami/mongodb:5.0.9-debian-10-r15
      imagePullPolicy: "IfNotPresent"
      securityContext:
        runAsNonRoot: true
        runAsUser: 1001
      env:
        - name: BITNAMI_DEBUG
          value: "false"
        - name: MONGODB_ROOT_USER
          value: "root"
        - name: MONGODB_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: mongodb-prom
              key: mongodb-root-password
        - name: ALLOW_EMPTY_PASSWORD
          value: "no"
        - name: MONGODB_SYSTEM_LOG_VERBOSITY
          value: "0"
        - name: MONGODB_DISABLE_SYSTEM_LOG
          value: "no"
        - name: MONGODB_DISABLE_JAVASCRIPT
          value: "no"
        - name: MONGODB_ENABLE_JOURNAL
          value: "yes"
        - name: MONGODB_PORT_NUMBER
          value: "27017"
        - name: MONGODB_ENABLE_IPV6
          value: "no"
        - name: MONGODB_ENABLE_DIRECTORY_PER_DB
          value: "no"
      ports:
        - name: mongodb
          containerPort: 27017
      
      volumeMounts:
        - name: datadir
          mountPath: /bitnami/mongodb
        - name: datadir
          mountPath: /tmp           
    - name: metrics
      image: docker.io/bitnami/mongodb-exporter:0.32.0-debian-11-r5
      imagePullPolicy: "IfNotPresent"
      securityContext:
        runAsNonRoot: true
        runAsUser: 1001
      command:
        - /bin/bash
        - -ec
      args:
        - |
          /bin/mongodb_exporter --web.listen-address ":9216" --mongodb.uri "mongodb://$MONGODB_ROOT_USER:$(echo $MONGODB_ROOT_PASSWORD | sed -r "s/@/%40/g;s/:/%3A/g")@localhost:27017/admin?" --collect-all --compatible-mode 
      env:
        - name: MONGODB_ROOT_USER
          value: "root"
        - name: MONGODB_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: mongodb-prom
              key: mongodb-root-password
      ports:
        - name: metrics
          containerPort: 9216
      livenessProbe:
        failureThreshold: 3
        initialDelaySeconds: 15
        periodSeconds: 5
        successThreshold: 1
        timeoutSeconds: 5
        httpGet:
          path: /
          port: metrics
      readinessProbe:
        failureThreshold: 3
        initialDelaySeconds: 5
        periodSeconds: 5
        successThreshold: 1
        timeoutSeconds: 1
        httpGet:
          path: /
          port: metrics
      resources:
        limits: {}
        requests: {}
  volumes:
    - name: datadir
      persistentVolumeClaim:
        claimName: mongodb
   

The solution is from the following thread:

GithubIssue

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.