2022-03-08T10:47:47.717566938Z level=info ts=2022-03-08T10:47:47.717045695Z caller=main.go:152 msg="Tracing will be disabled"
2022-03-08T10:47:47.717566938Z level=info ts=2022-03-08T10:47:47.717303607Z caller=client.go:54 msg="enabling client to server TLS"
2022-03-08T10:47:47.717673345Z level=info ts=2022-03-08T10:47:47.717526947Z caller=options.go:76 msg="TLS client using provided certificate pool"
2022-03-08T10:47:47.719100398Z level=info ts=2022-03-08T10:47:47.717788894Z caller=options.go:104 msg="TLS client authentication enabled"
2022-03-08T10:47:47.722566682Z level=info ts=2022-03-08T10:47:47.720533766Z caller=options.go:23 protocol=gRPC msg="disabled TLS, key and cert must be set to enable"
2022-03-08T10:47:47.722566682Z level=info ts=2022-03-08T10:47:47.720707427Z caller=grpc.go:93 service=gRPC/server component=query msg="registering as gRPC StoreAPI and RulesAPI"
2022-03-08T10:47:47.722566682Z level=info ts=2022-03-08T10:47:47.721339481Z caller=query.go:484 msg="starting query node"
2022-03-08T10:47:47.722595281Z level=info ts=2022-03-08T10:47:47.722554964Z caller=intrumentation.go:48 msg="changing probe status" status=ready
2022-03-08T10:47:47.726598931Z level=info ts=2022-03-08T10:47:47.72268529Z caller=intrumentation.go:60 msg="changing probe status" status=healthy
2022-03-08T10:47:47.726598931Z level=info ts=2022-03-08T10:47:47.724763009Z caller=http.go:57 service=http/server component=query msg="listening for requests and metrics" address=127.0.0.1:9090
2022-03-08T10:47:47.726598931Z level=info ts=2022-03-08T10:47:47.724884178Z caller=grpc.go:120 service=gRPC/server component=query msg="listening for serving gRPC" address=127.0.0.1:10901
2022-03-08T10:47:47.738350215Z level=error ts=2022-03-08T10:47:47.738227381Z caller=query.go:381 msg="failed to resolve addresses for storeAPIs" err="lookup SRV records \"_grpc._tcp.prometheus-operated.openshift-monitoring.svc.cluster.local\": lookup _grpc._tcp.prometheus-operated.openshift-monitoring.svc.cluster.local on 172.30.0.10:53: no such host"
2022-03-08T10:47:47.759838470Z level=error ts=2022-03-08T10:47:47.759746788Z caller=query.go:384 msg="failed to resolve addresses for rulesAPIs" err="lookup SRV records \"_grpc._tcp.prometheus-operated.openshift-monitoring.svc.cluster.local\": lookup _grpc._tcp.prometheus-operated.openshift-monitoring.svc.cluster.local on 172.30.0.10:53: no such host"
2022-03-08T10:47:58.206587593Z level=warn ts=2022-03-08T10:47:58.205556736Z caller=proxy.go:310 err="No StoreAPIs matched for this query" stores=
2022-03-08T10:47:58.206587593Z level=warn ts=2022-03-08T10:47:58.205641081Z caller=proxy.go:310 err="No StoreAPIs matched for this query" stores=
2022-03-08T10:48:16.804234219Z level=warn ts=2022-03-08T10:48:16.803911639Z caller=proxy.go:310 err="No StoreAPIs matched for this query" stores=
2022-03-08T10:48:16.804234219Z level=warn ts=2022-03-08T10:48:16.80396527Z caller=proxy.go:310 err="No StoreAPIs matched for this query" stores=
2022-03-08T10:48:22.737698725Z level=info ts=2022-03-08T10:48:22.737595078Z caller=storeset.go:416 component=storeset msg="adding new rulesAPI to query storeset" address=10.128.2.6:10901
2022-03-08T10:48:22.737698725Z level=info ts=2022-03-08T10:48:22.737638665Z caller=storeset.go:419 component=storeset msg="adding new storeAPI to query storeset" address=10.128.2.6:10901 extLset="{prometheus=\"openshift-monitoring/k8s\", prometheus_replica=\"prometheus-k8s-0\"}"
2022-03-08T10:48:22.737698725Z level=info ts=2022-03-08T10:48:22.737664649Z caller=storeset.go:416 component=storeset msg="adding new rulesAPI to query storeset" address=10.131.0.18:10901
2022-03-08T10:48:22.737698725Z level=info ts=2022-03-08T10:48:22.737675105Z caller=storeset.go:419 component=storeset msg="adding new storeAPI to query storeset" address=10.131.0.18:10901 extLset="{prometheus=\"openshift-monitoring/k8s\", prometheus_replica=\"prometheus-k8s-1\"}"
2022-03-08T11:01:27.723639487Z level=warn ts=2022-03-08T11:01:27.723379173Z caller=storeset.go:490 component=storeset msg="update of store node failed" err="getting metadata: fetching store info from 10.131.0.18:10901: rpc error: code = DeadlineExceeded desc = latest balancer error: connection error: desc = \"transport: Error while dialing dial tcp 10.131.0.18:10901: connect: connection refused\"" address=10.131.0.18:10901
2022-03-08T11:01:27.723739468Z level=info ts=2022-03-08T11:01:27.723624677Z caller=storeset.go:390 component=storeset msg="removing store because it's unhealthy or does not exist" address=10.131.0.18:10901 extLset="{prometheus=\"openshift-monitoring/k8s\", prometheus_replica=\"prometheus-k8s-1\"}"
2022-03-08T11:01:32.724153505Z level=warn ts=2022-03-08T11:01:32.724033803Z caller=storeset.go:490 component=storeset msg="update of store node failed" err="getting metadata: fetching store info from 10.131.0.18:10901: rpc error: code = DeadlineExceeded desc = context deadline exceeded" address=10.131.0.18:10901
2022-03-08T11:01:37.724542563Z level=warn ts=2022-03-08T11:01:37.724425706Z caller=storeset.go:490 component=storeset msg="update of store node failed" err="getting metadata: fetching store info from 10.131.0.18:10901: rpc error: code = DeadlineExceeded desc = context deadline exceeded" address=10.131.0.18:10901
2022-03-08T11:01:42.725336289Z level=warn ts=2022-03-08T11:01:42.724925988Z caller=storeset.go:490 component=storeset msg="update of store node failed" err="getting metadata: fetching store info from 10.131.0.18:10901: rpc error: code = DeadlineExceeded desc = context deadline exceeded" address=10.131.0.18:10901
2022-03-08T11:01:47.725406118Z level=warn ts=2022-03-08T11:01:47.725311749Z caller=storeset.go:490 component=storeset msg="update of store node failed" err="getting metadata: fetching store info from 10.131.0.18:10901: rpc error: code = DeadlineExceeded desc = context deadline exceeded" address=10.131.0.18:10901
2022-03-08T11:01:47.735901440Z level=info ts=2022-03-08T11:01:47.735613351Z caller=storeset.go:390 component=storeset msg="removing store because it's unhealthy or does not exist" address=10.128.2.6:10901 extLset="{prometheus=\"openshift-monitoring/k8s\", prometheus_replica=\"prometheus-k8s-0\"}"
2022-03-08T11:01:47.735901440Z level=info ts=2022-03-08T11:01:47.73567489Z caller=storeset.go:416 component=storeset msg="adding new rulesAPI to query storeset" address=10.129.2.15:10901
2022-03-08T11:01:47.735901440Z level=info ts=2022-03-08T11:01:47.735687229Z caller=storeset.go:419 component=storeset msg="adding new storeAPI to query storeset" address=10.129.2.15:10901 extLset="{prometheus=\"openshift-monitoring/k8s\", prometheus_replica=\"prometheus-k8s-1\"}"
2022-03-08T11:02:22.732630283Z level=info ts=2022-03-08T11:02:22.732476368Z caller=storeset.go:416 component=storeset msg="adding new rulesAPI to query storeset" address=10.128.2.7:10901
2022-03-08T11:02:22.732630283Z level=info ts=2022-03-08T11:02:22.732514223Z caller=storeset.go:419 component=storeset msg="adding new storeAPI to query storeset" address=10.128.2.7:10901 extLset="{prometheus=\"openshift-monitoring/k8s\", prometheus_replica=\"prometheus-k8s-0\"}"
