---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  creationTimestamp: "2022-03-08T10:48:10Z"
  generation: 1
  labels:
    app.kubernetes.io/component: query-layer
    app.kubernetes.io/instance: thanos-querier
    app.kubernetes.io/name: thanos-query
    app.kubernetes.io/version: 0.12.0
  managedFields:
  - apiVersion: monitoring.coreos.com/v1
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:labels:
          .: {}
          f:app.kubernetes.io/component: {}
          f:app.kubernetes.io/instance: {}
          f:app.kubernetes.io/name: {}
          f:app.kubernetes.io/version: {}
      f:spec:
        .: {}
        f:groups: {}
    manager: operator
    operation: Update
    time: "2022-03-08T10:48:10Z"
  name: thanos-querier
  namespace: openshift-monitoring
  resourceVersion: "21519"
  selfLink: /apis/monitoring.coreos.com/v1/namespaces/openshift-monitoring/prometheusrules/thanos-querier
  uid: 76e1f066-493f-4d5f-87cb-c8aeed891edf
spec:
  groups:
  - name: thanos-query.rules
    rules:
    - alert: ThanosQueryHttpRequestQueryErrorRateHigh
      annotations:
        message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query" requests.
      expr: |
        (
          sum(rate(http_requests_total{code=~"5..", job="thanos-querier", handler="query"}[5m]))
        /
          sum(rate(http_requests_total{job="thanos-querier", handler="query"}[5m]))
        ) * 100 > 5
      for: 5m
      labels:
        severity: critical
    - alert: ThanosQueryHttpRequestQueryRangeErrorRateHigh
      annotations:
        message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of "query_range" requests.
      expr: |
        (
          sum(rate(http_requests_total{code=~"5..", job="thanos-querier", handler="query_range"}[5m]))
        /
          sum(rate(http_requests_total{job="thanos-querier", handler="query_range"}[5m]))
        ) * 100 > 5
      for: 5m
      labels:
        severity: critical
    - alert: ThanosQueryGrpcServerErrorRate
      annotations:
        message: Thanos Query {{$labels.job}} is failing to handle {{ $value | humanize }}% of requests.
      expr: |
        (
          sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job="thanos-querier"}[5m]))
        /
          sum by (job) (rate(grpc_server_started_total{job="thanos-querier"}[5m]))
        * 100 > 5
        )
      for: 5m
      labels:
        severity: warning
    - alert: ThanosQueryGrpcClientErrorRate
      annotations:
        message: Thanos Query {{$labels.job}} is failing to send {{ $value | humanize }}% of requests.
      expr: |
        (
          sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", job="thanos-querier"}[5m]))
        /
          sum by (job) (rate(grpc_client_started_total{job="thanos-querier"}[5m]))
        ) * 100 > 5
      for: 5m
      labels:
        severity: warning
    - alert: ThanosQueryHighDNSFailures
      annotations:
        message: Thanos Query {{$labels.job}} have {{ $value | humanize }}% of failing DNS queries for store endpoints.
      expr: |
        (
          sum by (job) (rate(thanos_querier_store_apis_dns_failures_total{job="thanos-querier"}[5m]))
        /
          sum by (job) (rate(thanos_querier_store_apis_dns_lookups_total{job="thanos-querier"}[5m]))
        ) * 100 > 1
      for: 15m
      labels:
        severity: warning
    - alert: ThanosQueryInstantLatencyHigh
      annotations:
        message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for instant queries.
      expr: |
        (
          histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job="thanos-querier", handler="query"}[5m]))) > 40
        and
          sum by (job) (rate(http_request_duration_seconds_bucket{job="thanos-querier", handler="query"}[5m])) > 0
        )
      for: 10m
      labels:
        severity: critical
    - alert: ThanosQueryRangeLatencyHigh
      annotations:
        message: Thanos Query {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for range queries.
      expr: |
        (
          histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job="thanos-querier", handler="query_range"}[5m]))) > 90
        and
          sum by (job) (rate(http_request_duration_seconds_count{job="thanos-querier", handler="query_range"}[5m])) > 0
        )
      for: 10m
      labels:
        severity: critical
