2022-03-08T11:02:03.241323050Z level=info ts=2022-03-08T11:02:03.240Z caller=main.go:353 msg="Starting Prometheus" version="(version=2.22.2, branch=rhaos-4.6-rhel-8, revision=c1220a31bb37bd2dcffcbf1b00b8f63f55032231)"
2022-03-08T11:02:03.241323050Z level=info ts=2022-03-08T11:02:03.240Z caller=main.go:358 build_context="(go=go1.15.14, user=root@cb22954a3573, date=20211130-18:26:30)"
2022-03-08T11:02:03.241323050Z level=info ts=2022-03-08T11:02:03.240Z caller=main.go:359 host_details="(Linux 4.18.0-193.70.1.el8_2.x86_64 #1 SMP Mon Nov 15 00:34:11 EST 2021 x86_64 prometheus-k8s-0 (none))"
2022-03-08T11:02:03.241323050Z level=info ts=2022-03-08T11:02:03.240Z caller=main.go:360 fd_limits="(soft=1048576, hard=1048576)"
2022-03-08T11:02:03.241323050Z level=info ts=2022-03-08T11:02:03.240Z caller=main.go:361 vm_limits="(soft=unlimited, hard=unlimited)"
2022-03-08T11:02:03.244343104Z level=info ts=2022-03-08T11:02:03.244Z caller=main.go:712 msg="Starting TSDB ..."
2022-03-08T11:02:03.244390646Z level=info ts=2022-03-08T11:02:03.244Z caller=web.go:516 component=web msg="Start listening for connections" address=127.0.0.1:9090
2022-03-08T11:02:03.247953520Z level=info ts=2022-03-08T11:02:03.247Z caller=head.go:642 component=tsdb msg="Replaying on-disk memory mappable chunks if any"
2022-03-08T11:02:03.247953520Z level=info ts=2022-03-08T11:02:03.247Z caller=head.go:656 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.078µs
2022-03-08T11:02:03.247953520Z level=info ts=2022-03-08T11:02:03.247Z caller=head.go:662 component=tsdb msg="Replaying WAL, this may take a while"
2022-03-08T11:02:03.247953520Z level=info ts=2022-03-08T11:02:03.247Z caller=head.go:714 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
2022-03-08T11:02:03.247953520Z level=info ts=2022-03-08T11:02:03.247Z caller=head.go:719 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.267µs wal_replay_duration=151.134µs total_replay_duration=185.259µs
2022-03-08T11:02:03.249209947Z level=info ts=2022-03-08T11:02:03.248Z caller=main.go:732 fs_type=XFS_SUPER_MAGIC
2022-03-08T11:02:03.249209947Z level=info ts=2022-03-08T11:02:03.248Z caller=main.go:735 msg="TSDB started"
2022-03-08T11:02:03.249209947Z level=info ts=2022-03-08T11:02:03.248Z caller=main.go:861 msg="Loading configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml
2022-03-08T11:02:03.268297484Z level=info ts=2022-03-08T11:02:03.267Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.270763588Z level=info ts=2022-03-08T11:02:03.270Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.272055936Z level=info ts=2022-03-08T11:02:03.272Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.273377558Z level=info ts=2022-03-08T11:02:03.273Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.274579991Z level=info ts=2022-03-08T11:02:03.274Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.275766194Z level=info ts=2022-03-08T11:02:03.275Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.277502379Z level=info ts=2022-03-08T11:02:03.277Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.278616850Z level=info ts=2022-03-08T11:02:03.278Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.280620689Z level=info ts=2022-03-08T11:02:03.280Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.281720362Z level=info ts=2022-03-08T11:02:03.281Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.283810300Z level=info ts=2022-03-08T11:02:03.283Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.285205983Z level=info ts=2022-03-08T11:02:03.285Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.286531919Z level=info ts=2022-03-08T11:02:03.286Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.289067910Z level=info ts=2022-03-08T11:02:03.288Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.290883783Z level=info ts=2022-03-08T11:02:03.290Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.293359436Z level=info ts=2022-03-08T11:02:03.292Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.295314954Z level=info ts=2022-03-08T11:02:03.295Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.298291862Z level=info ts=2022-03-08T11:02:03.297Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.299263264Z level=info ts=2022-03-08T11:02:03.299Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.300705075Z level=info ts=2022-03-08T11:02:03.300Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.302076664Z level=info ts=2022-03-08T11:02:03.302Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.303559080Z level=info ts=2022-03-08T11:02:03.303Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.304897869Z level=info ts=2022-03-08T11:02:03.304Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.306164960Z level=info ts=2022-03-08T11:02:03.306Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.307652324Z level=info ts=2022-03-08T11:02:03.307Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.309048941Z level=info ts=2022-03-08T11:02:03.308Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.310333374Z level=info ts=2022-03-08T11:02:03.310Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.311670113Z level=info ts=2022-03-08T11:02:03.311Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.312975684Z level=info ts=2022-03-08T11:02:03.312Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.315022539Z level=info ts=2022-03-08T11:02:03.314Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.315768685Z level=info ts=2022-03-08T11:02:03.315Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.317167600Z level=info ts=2022-03-08T11:02:03.317Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.320272781Z level=info ts=2022-03-08T11:02:03.319Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.322284104Z level=info ts=2022-03-08T11:02:03.321Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.324104719Z level=info ts=2022-03-08T11:02:03.324Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.325531722Z level=info ts=2022-03-08T11:02:03.325Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.328297419Z level=info ts=2022-03-08T11:02:03.327Z caller=kubernetes.go:263 component="discovery manager notify" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:03.422357338Z level=info ts=2022-03-08T11:02:03.421Z caller=main.go:892 msg="Completed loading of configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml totalDuration=173.049012ms remote_storage=3.323µs web_handler=638ns query_engine=1.491µs scrape=385.717µs scrape_sd=59.888186ms notify=153.905µs notify_sd=1.425764ms rules=93.126352ms
2022-03-08T11:02:03.422357338Z level=info ts=2022-03-08T11:02:03.421Z caller=main.go:684 msg="Server is ready to receive web requests."
2022-03-08T11:02:06.069463692Z level=info ts=2022-03-08T11:02:06.068Z caller=main.go:861 msg="Loading configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml
2022-03-08T11:02:06.094529628Z level=warn ts=2022-03-08T11:02:06.093Z caller=klog.go:88 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:426: watch of *v1.Endpoints ended with: an error on the server (\"unable to decode an event from the watch stream: context canceled\") has prevented the request from succeeding"
2022-03-08T11:02:06.094529628Z level=info ts=2022-03-08T11:02:06.093Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.095290495Z level=warn ts=2022-03-08T11:02:06.095Z caller=klog.go:88 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:428: watch of *v1.Pod ended with: an error on the server (\"unable to decode an event from the watch stream: context canceled\") has prevented the request from succeeding"
2022-03-08T11:02:06.096267136Z level=warn ts=2022-03-08T11:02:06.095Z caller=klog.go:88 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:428: watch of *v1.Pod ended with: an error on the server (\"unable to decode an event from the watch stream: context canceled\") has prevented the request from succeeding"
2022-03-08T11:02:06.096267136Z level=warn ts=2022-03-08T11:02:06.096Z caller=klog.go:88 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:428: watch of *v1.Pod ended with: an error on the server (\"unable to decode an event from the watch stream: context canceled\") has prevented the request from succeeding"
2022-03-08T11:02:06.096267136Z level=info ts=2022-03-08T11:02:06.096Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.097254541Z level=warn ts=2022-03-08T11:02:06.096Z caller=klog.go:88 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:427: watch of *v1.Service ended with: an error on the server (\"unable to decode an event from the watch stream: context canceled\") has prevented the request from succeeding"
2022-03-08T11:02:06.098262629Z level=warn ts=2022-03-08T11:02:06.097Z caller=klog.go:88 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:427: watch of *v1.Service ended with: an error on the server (\"unable to decode an event from the watch stream: context canceled\") has prevented the request from succeeding"
2022-03-08T11:02:06.098262629Z level=warn ts=2022-03-08T11:02:06.097Z caller=klog.go:88 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:427: watch of *v1.Service ended with: an error on the server (\"unable to decode an event from the watch stream: context canceled\") has prevented the request from succeeding"
2022-03-08T11:02:06.099250027Z level=info ts=2022-03-08T11:02:06.098Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.100327099Z level=warn ts=2022-03-08T11:02:06.099Z caller=klog.go:88 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:428: watch of *v1.Pod ended with: an error on the server (\"unable to decode an event from the watch stream: context canceled\") has prevented the request from succeeding"
2022-03-08T11:02:06.100595083Z level=warn ts=2022-03-08T11:02:06.100Z caller=klog.go:88 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:428: watch of *v1.Pod ended with: an error on the server (\"unable to decode an event from the watch stream: context canceled\") has prevented the request from succeeding"
2022-03-08T11:02:06.103513457Z level=info ts=2022-03-08T11:02:06.102Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.104293170Z level=info ts=2022-03-08T11:02:06.104Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.105467524Z level=info ts=2022-03-08T11:02:06.105Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.106679736Z level=info ts=2022-03-08T11:02:06.106Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.107915748Z level=info ts=2022-03-08T11:02:06.107Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.109090632Z level=info ts=2022-03-08T11:02:06.109Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.110263969Z level=info ts=2022-03-08T11:02:06.110Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.111418609Z level=info ts=2022-03-08T11:02:06.111Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.112592234Z level=info ts=2022-03-08T11:02:06.112Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.113853599Z level=info ts=2022-03-08T11:02:06.113Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.115288460Z level=info ts=2022-03-08T11:02:06.115Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.116410148Z level=info ts=2022-03-08T11:02:06.116Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.117616241Z level=info ts=2022-03-08T11:02:06.117Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.118824569Z level=info ts=2022-03-08T11:02:06.118Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.120013319Z level=info ts=2022-03-08T11:02:06.119Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.121157383Z level=info ts=2022-03-08T11:02:06.121Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.122418863Z level=info ts=2022-03-08T11:02:06.122Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.123631985Z level=info ts=2022-03-08T11:02:06.123Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.124865894Z level=info ts=2022-03-08T11:02:06.124Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.126023057Z level=info ts=2022-03-08T11:02:06.125Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.127181954Z level=info ts=2022-03-08T11:02:06.127Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.128446950Z level=info ts=2022-03-08T11:02:06.128Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.129678575Z level=info ts=2022-03-08T11:02:06.129Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.130936316Z level=info ts=2022-03-08T11:02:06.130Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.132087904Z level=info ts=2022-03-08T11:02:06.132Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.133265842Z level=info ts=2022-03-08T11:02:06.133Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.134438958Z level=info ts=2022-03-08T11:02:06.134Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.135731632Z level=info ts=2022-03-08T11:02:06.135Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.137149381Z level=info ts=2022-03-08T11:02:06.137Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.138471468Z level=info ts=2022-03-08T11:02:06.138Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.141299465Z level=info ts=2022-03-08T11:02:06.141Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.144299104Z level=info ts=2022-03-08T11:02:06.143Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.146251272Z level=info ts=2022-03-08T11:02:06.145Z caller=kubernetes.go:263 component="discovery manager scrape" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.148293547Z level=info ts=2022-03-08T11:02:06.147Z caller=kubernetes.go:263 component="discovery manager notify" discovery=kubernetes msg="Using pod service account via in-cluster config"
2022-03-08T11:02:06.251410032Z level=info ts=2022-03-08T11:02:06.251Z caller=main.go:892 msg="Completed loading of configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml totalDuration=183.279193ms remote_storage=3.623µs web_handler=4.642µs query_engine=2.159µs scrape=140.053µs scrape_sd=53.752209ms notify=155.246µs notify_sd=1.347845ms rules=102.860893ms
