---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  creationTimestamp: "2022-03-08T10:14:06Z"
  generation: 1
  labels:
    k8s-app: machine-config-daemon
  managedFields:
  - apiVersion: monitoring.coreos.com/v1
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:labels:
          .: {}
          f:k8s-app: {}
      f:spec:
        .: {}
        f:groups: {}
    manager: cluster-version-operator
    operation: Update
    time: "2022-03-08T10:14:06Z"
  name: machine-config-daemon
  namespace: openshift-machine-config-operator
  resourceVersion: "1428"
  selfLink: /apis/monitoring.coreos.com/v1/namespaces/openshift-machine-config-operator/prometheusrules/machine-config-daemon
  uid: a79412fd-6b4c-4b7b-ba44-a997f3422802
spec:
  groups:
  - name: mcd-reboot-error
    rules:
    - alert: MCDRebootError
      annotations:
        message: Reboot failed on {{ $labels.node }} , update may be blocked
      expr: |
        mcd_reboot_err > 0
      labels:
        severity: critical
  - name: mcd-drain-error
    rules:
    - alert: MCDDrainError
      annotations:
        message: 'Drain failed on {{ $labels.node }} , updates may be blocked. For more details:  oc logs -f -n openshift-machine-config-operator machine-config-daemon-<hash> -c machine-config-daemon'
      expr: |
        mcd_drain_err > 0
      labels:
        severity: warning
  - name: mcd-pivot-error
    rules:
    - alert: MCDPivotError
      annotations:
        message: 'Error detected in pivot logs on {{ $labels.node }} '
      expr: |
        mcd_pivot_err > 0
      labels:
        severity: warning
  - name: mcd-kubelet-health-state-error
    rules:
    - alert: KubeletHealthState
      annotations:
        message: Kubelet health failure threshold reached
      expr: |
        mcd_kubelet_state > 2
      labels:
        severity: warning
  - name: system-memory-exceeds-reservation
    rules:
    - alert: SystemMemoryExceedsReservation
      annotations:
        message: System memory usage of {{ $value | humanize }} on {{ $labels.node }} exceeds 95% of the reservation. Reserved memory ensures system processes can function even when the node is fully allocated and protects against workload out of memory events impacting the proper functioning of the node. The default reservation is expected to be sufficient for most configurations and should be increased (https://docs.openshift.com/container-platform/latest/nodes/nodes/nodes-nodes-managing.html) when running nodes with high numbers of pods (either due to rate of change or at steady state).
      expr: |
        sum by (node) (container_memory_rss{id="/system.slice"}) > ((sum by (node) (kube_node_status_capacity{resource="memory"} - kube_node_status_allocatable{resource="memory"})) * 0.95)
      for: 15m
      labels:
        severity: warning
