[notice] 14#14: using the "epoll" event method [notice] 14#14: openresty/1.15.8.1 [notice] 14#14: built by gcc 4.8.5 20150623 (Red Hat 4.8.5-36) (GCC) [notice] 14#14: OS: Linux 4.18.0-80.11.2.el8_0.x86_64 [notice] 14#14: getrlimit(RLIMIT_NOFILE): 1048576:1048576 [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_VALIDATE_POLICY_CONFIGS = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: TEST_NGINX_BINARY = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: OPENSSL_VERIFY = false [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_POLICY_LOAD_PATH = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_DIR = /opt/app-root/src [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_BUILTIN_POLICY_LOAD_PATH = /opt/app-root/src/src/apicast/policy [debug] 14#14: *2 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 14#14: *2 [lua] policy_loader.lua:150: loading policy: load_configuration version: builtin [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_PATH_ROUTING = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_PATH_ROUTING_ENABLED = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_PATH_ROUTING_ONLY = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: all_proxy = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: ALL_PROXY = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: http_proxy = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: HTTP_PROXY = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: https_proxy = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: HTTPS_PROXY = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: no_proxy = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: NO_PROXY = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: BACKEND_ENDPOINT_OVERRIDE = http://backend-listener:3000 [debug] 14#14: *2 [lua] evp.lua:142: openssl11=true err=nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: PWD = /opt/app-root [warn] 14#14: *2 [lua] _G write guard:12: writing a global lua variable ('yaml') which may lead to race conditions between concurrent requests, so prefer the use of 'local' variables stack traceback: [C]: at 0x7f3bf3f0ed00 [C]: in function 'require' /usr/local/share/lua/5.1/lyaml/init.lua:34: in main chunk [C]: in function 'require' /opt/app-root/src/src/resty/yaml.lua:6: in main chunk [C]: in function 'require' ...p-root/src/src/apicast/configuration_loader/data_url.lua:9: in main chunk [C]: in function 'require' /opt/app-root/src/src/apicast/configuration_loader.lua:5: in main chunk [C]: in function 'root_require' ... [C]: in function 'require' /opt/app-root/src/src/apicast/cli/command/start.lua:18: in main chunk [C]: in function 'require' /opt/app-root/src/src/apicast/cli.lua:18: in function 'load_commands' /opt/app-root/src/src/apicast/cli.lua:23: in main chunk [C]: in function 'require' /tmp/Oyx_4IkzaM:60: in function 'file_gen' init_worker_by_lua:51: in function [C]: in function 'xpcall' init_worker_by_lua:58: in function , context: ngx.timer [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_OIDC_LOG_LEVEL = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_CONFIGURATION_LOADER = lazy [info] 14#14: *2 [lua] configuration_loader.lua:239: new(): using lazy configuration loader, context: ngx.timer [debug] 14#14: *2 [lua] policy_loader.lua:150: loading policy: find_service version: builtin [debug] 14#14: *2 [lua] find_service.lua:23: find_service_func(): Using host-based routing [debug] 14#14: *2 [lua] policy_loader.lua:150: loading policy: local_chain version: builtin [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_CUSTOM_CONFIG = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_RESPONSE_CODES = true [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_REPORTING_THREADS = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_MODULE = nil [debug] 14#14: *2 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 14#14: *2 [lua] policy_loader.lua:150: loading policy: nginx_metrics version: builtin [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_EXTENDED_METRICS = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: NGINX_METRICS_MAX_LOGS = nil [debug] 14#14: *2 [lua] resolver.lua:114: parse_nameservers(): /etc/resolv.conf: search threescale-postgres.svc.cluster.local svc.cluster.local cluster.local us-east-2.compute.internal nameserver 172.30.0.10 options ndots:5 [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: RESOLVER = nil [debug] 14#14: *2 [lua] resolver.lua:135: parse_nameservers(): search threescale-postgres.svc.cluster.local svc.cluster.local cluster.local us-east-2.compute.internal [debug] 14#14: *2 [lua] resolver.lua:139: parse_nameservers(): search domain: threescale-postgres.svc.cluster.local [debug] 14#14: *2 [lua] resolver.lua:139: parse_nameservers(): search domain: svc.cluster.local [debug] 14#14: *2 [lua] resolver.lua:139: parse_nameservers(): search domain: cluster.local [debug] 14#14: *2 [lua] resolver.lua:139: parse_nameservers(): search domain: us-east-2.compute.internal [info] 14#14: *2 [lua] resolver.lua:160: init_nameservers(): adding 172.30.0.10:53 as default nameserver, context: ngx.timer [info] 14#14: *2 [lua] resolver.lua:165: init_nameservers(): adding threescale-postgres.svc.cluster.local as search domain, context: ngx.timer [info] 14#14: *2 [lua] resolver.lua:165: init_nameservers(): adding svc.cluster.local as search domain, context: ngx.timer [info] 14#14: *2 [lua] resolver.lua:165: init_nameservers(): adding cluster.local as search domain, context: ngx.timer [info] 14#14: *2 [lua] resolver.lua:165: init_nameservers(): adding us-east-2.compute.internal as search domain, context: ngx.timer [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: THREESCALE_DEPLOYMENT_ENV = staging [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_LOADED_ENVIRONMENTS = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_ENVIRONMENT = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_CONFIGURATION = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_WORKERS = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_CONFIGURATION_CACHE = 0 [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_LOG_LEVEL = debug [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_LOG_FILE = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_ACCESS_LOG_FILE = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_OPENRESTY_BINARY = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: SSL_CERT_FILE = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: OPENTRACING_TRACER = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: HTTP_KEEPALIVE_TIMEOUT = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: OPENTRACING_FORWARD_HEADER = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_PROXY_HTTPS_CERTIFICATE = nil [debug] 14#14: *2 [lua] env.lua:86: fetch(): env: APICAST_UPSTREAM_RETRY_CASES = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: THREESCALE_OPERATOR_PORT = tcp://172.30.97.220:8383 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_DIR = /opt/app-root/src [debug] 14#14: [lua] env.lua:86: fetch(): env: THREESCALE_DEPLOYMENT_ENV = staging [debug] 14#14: [lua] env.lua:86: fetch(): env: THREESCALE_OPERATOR_SERVICE_HOST = 172.30.97.220 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_PORT_8080_TCP_PORT = 8080 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_SERVICE_HOST = 172.30.151.209 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_RESPONSE_CODES = true [debug] 14#14: [lua] env.lua:86: fetch(): env: THREESCALE_OPERATOR_PORT_8383_TCP_PROTO = tcp [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_POLICY_LOAD_PATH = /opt/app-root/src/policies [debug] 14#14: [lua] env.lua:86: fetch(): env: THREESCALE_OPERATOR_PORT_8383_TCP = tcp://172.30.97.220:8383 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_SERVICE_HOST = 172.30.39.144 [debug] 14#14: [lua] env.lua:86: fetch(): env: THREESCALE_PORTAL_ENDPOINT = http://DXPhCB0j@system-master:3000/master/api/proxy/configs [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_SERVICE_PORT_GATEWAY = 8080 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_PORT = tcp://172.30.39.144:8080 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_PORT_8080_TCP_PROTO = tcp [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_PORT_8080_TCP_PORT = 8080 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_PORT_8090_TCP = tcp://172.30.151.209:8090 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_LOG_LEVEL = debug [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_SERVICE_PORT_MANAGEMENT = 8090 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_PORT_8090_TCP_PORT = 8090 [debug] 14#14: [lua] env.lua:86: fetch(): env: THREESCALE_OPERATOR_PORT_8383_TCP_PORT = 8383 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_PORT_8080_TCP_ADDR = 172.30.151.209 [debug] 14#14: [lua] env.lua:86: fetch(): env: THREESCALE_OPERATOR_SERVICE_PORT = 8383 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_PORT_8080_TCP_PROTO = tcp [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_PORT_8090_TCP_ADDR = 172.30.39.144 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_PORT_8080_TCP = tcp://172.30.151.209:8080 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_SERVICE_PORT_GATEWAY = 8080 [debug] 14#14: [lua] env.lua:86: fetch(): env: THREESCALE_OPERATOR_PORT_8383_TCP_ADDR = 172.30.97.220 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_PORT_8090_TCP_PROTO = tcp [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_PORT = tcp://172.30.151.209:8080 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_PORT_8080_TCP_ADDR = 172.30.39.144 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_CONFIGURATION_CACHE = 0 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_SERVICE_PORT_MANAGEMENT = 8090 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_PORT_8090_TCP = tcp://172.30.39.144:8090 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_PORT_8090_TCP_PROTO = tcp [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_BUILTIN_POLICY_LOAD_PATH = /opt/app-root/src/src/apicast/policy [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_LOADED_ENVIRONMENTS = /opt/app-root/src/config/staging.lua [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_SERVICE_PORT = 8080 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_SERVICE_PORT = 8080 [debug] 14#14: [lua] env.lua:86: fetch(): env: THREESCALE_OPERATOR_SERVICE_PORT_METRICS = 8383 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_CONFIGURATION_LOADER = lazy [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_STAGING_PORT_8090_TCP_ADDR = 172.30.151.209 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_PORT_8080_TCP = tcp://172.30.39.144:8080 [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_MANAGEMENT_API = status [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PRODUCTION_PORT_8090_TCP_PORT = 8090 [debug] 14#14: [lua] resolver.lua:114: parse_nameservers(): /etc/resolv.conf: search threescale-postgres.svc.cluster.local svc.cluster.local cluster.local us-east-2.compute.internal nameserver 172.30.0.10 options ndots:5 [debug] 14#14: [lua] env.lua:86: fetch(): env: RESOLVER = nil [debug] 14#14: [lua] resolver.lua:135: parse_nameservers(): search threescale-postgres.svc.cluster.local svc.cluster.local cluster.local us-east-2.compute.internal [debug] 14#14: [lua] resolver.lua:139: parse_nameservers(): search domain: threescale-postgres.svc.cluster.local [debug] 14#14: [lua] resolver.lua:139: parse_nameservers(): search domain: svc.cluster.local [debug] 14#14: [lua] resolver.lua:139: parse_nameservers(): search domain: cluster.local [debug] 14#14: [lua] resolver.lua:139: parse_nameservers(): search domain: us-east-2.compute.internal [info] 14#14: [lua] resolver.lua:160: init_nameservers(): adding 172.30.0.10:53 as default nameserver [info] 14#14: [lua] resolver.lua:165: init_nameservers(): adding threescale-postgres.svc.cluster.local as search domain [info] 14#14: [lua] resolver.lua:165: init_nameservers(): adding svc.cluster.local as search domain [info] 14#14: [lua] resolver.lua:165: init_nameservers(): adding cluster.local as search domain [info] 14#14: [lua] resolver.lua:165: init_nameservers(): adding us-east-2.compute.internal as search domain [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_VALIDATE_POLICY_CONFIGS = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: TEST_NGINX_BINARY = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: OPENSSL_VERIFY = false [debug] 14#14: [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 14#14: [lua] policy_loader.lua:150: loading policy: load_configuration version: builtin [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PATH_ROUTING = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PATH_ROUTING_ENABLED = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_PATH_ROUTING_ONLY = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: all_proxy = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: ALL_PROXY = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: http_proxy = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: HTTP_PROXY = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: https_proxy = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: HTTPS_PROXY = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: no_proxy = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: NO_PROXY = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: BACKEND_ENDPOINT_OVERRIDE = http://backend-listener:3000 [debug] 14#14: [lua] evp.lua:142: openssl11=true err=nil [debug] 14#14: [lua] env.lua:86: fetch(): env: PWD = /opt/app-root [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_OIDC_LOG_LEVEL = nil [info] 14#14: [lua] configuration_loader.lua:239: new(): using lazy configuration loader [debug] 14#14: [lua] policy_loader.lua:150: loading policy: find_service version: builtin [debug] 14#14: [lua] find_service.lua:23: find_service_func(): Using host-based routing [debug] 14#14: [lua] policy_loader.lua:150: loading policy: local_chain version: builtin [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_CUSTOM_CONFIG = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_REPORTING_THREADS = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_MODULE = nil [debug] 14#14: [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 14#14: [lua] policy_loader.lua:150: loading policy: nginx_metrics version: builtin [debug] 14#14: [lua] env.lua:86: fetch(): env: APICAST_EXTENDED_METRICS = nil [debug] 14#14: [lua] env.lua:86: fetch(): env: NGINX_METRICS_MAX_LOGS = nil [debug] 14#14: [lua] resolver.lua:114: parse_nameservers(): /etc/resolv.conf: search threescale-postgres.svc.cluster.local svc.cluster.local cluster.local us-east-2.compute.internal nameserver 172.30.0.10 options ndots:5 [debug] 14#14: [lua] resolver.lua:135: parse_nameservers(): search threescale-postgres.svc.cluster.local svc.cluster.local cluster.local us-east-2.compute.internal [debug] 14#14: [lua] resolver.lua:139: parse_nameservers(): search domain: threescale-postgres.svc.cluster.local [debug] 14#14: [lua] resolver.lua:139: parse_nameservers(): search domain: svc.cluster.local [debug] 14#14: [lua] resolver.lua:139: parse_nameservers(): search domain: cluster.local [debug] 14#14: [lua] resolver.lua:139: parse_nameservers(): search domain: us-east-2.compute.internal [info] 14#14: [lua] resolver.lua:160: init_nameservers(): adding 172.30.0.10:53 as default nameserver [info] 14#14: [lua] resolver.lua:165: init_nameservers(): adding threescale-postgres.svc.cluster.local as search domain [info] 14#14: [lua] resolver.lua:165: init_nameservers(): adding svc.cluster.local as search domain [info] 14#14: [lua] resolver.lua:165: init_nameservers(): adding cluster.local as search domain [info] 14#14: [lua] resolver.lua:165: init_nameservers(): adding us-east-2.compute.internal as search domain [notice] 14#14: [lua] environment.lua:199: add(): loading environment configuration: /opt/app-root/src/config/staging.lua [debug] 14#14: [lua] policy_loader.lua:150: loading policy: load_configuration version: builtin [debug] 14#14: [lua] policy_loader.lua:150: loading policy: find_service version: builtin [debug] 14#14: [lua] find_service.lua:23: find_service_func(): Using host-based routing [debug] 14#14: [lua] policy_loader.lua:150: loading policy: local_chain version: builtin [debug] 14#14: [lua] policy_loader.lua:150: loading policy: nginx_metrics version: builtin [debug] 14#14: [lua] executor.lua:26: init(): executor phase: init [debug] 14#14: [lua] policy_chain.lua:199: init(): policy chain execute phase: init, policy: Load Configuration, i: 1 [debug] 14#14: [lua] policy_chain.lua:199: init(): policy chain execute phase: init, policy: Find Service Policy, i: 2 [debug] 14#14: [lua] policy_chain.lua:199: init(): policy chain execute phase: init, policy: Local Policy Chain, i: 3 [debug] 14#14: [lua] policy_chain.lua:199: policy chain execute phase: init, policy: APIcast, i: 1 [debug] 14#14: [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 14#14: [lua] policy_chain.lua:199: init(): policy chain execute phase: init, policy: Metrics, i: 4 [debug] 14#14: [lua] env.lua:86: fetch(): env: NGINX_METRICS_LOG_LEVEL = nil [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: token_introspection version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: conditional version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: upstream version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: retry version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: default_credentials version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: cors version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: jwt_claim_check version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: logging version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: soap version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: liquid_context_debug version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: url_rewriting version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: 3scale_batcher version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: upstream_connection version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: tls_validation version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: tls version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: caching version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: rewrite_url_captures version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: rate_limit version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: oauth_mtls version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: apicast version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: 3scale_referrer version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: http_proxy version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: maintenance_mode version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: echo version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: headers version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: routing version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: ip_check version: builtin [debug] 14#14: [lua] policy_loader.lua:150: call(): loading policy: keycloak_role_check version: builtin [notice] 14#14: using the "epoll" event method [notice] 14#14: openresty/1.15.8.1 [notice] 14#14: built by gcc 4.8.5 20150623 (Red Hat 4.8.5-36) (GCC) [notice] 14#14: OS: Linux 4.18.0-80.11.2.el8_0.x86_64 [notice] 14#14: getrlimit(RLIMIT_NOFILE): 1048576:1048576 [notice] 14#14: start worker processes [notice] 14#14: start worker process 19 [notice] 14#14: signal 17 (SIGCHLD) received from 15 [debug] 19#19: *1 [lua] executor.lua:26: init_worker(): executor phase: init_worker [debug] 19#19: *1 [lua] policy_chain.lua:199: init_worker(): policy chain execute phase: init_worker, policy: Load Configuration, i: 1 [debug] 19#19: *1 [lua] policy_chain.lua:199: init_worker(): policy chain execute phase: init_worker, policy: Find Service Policy, i: 2 [debug] 19#19: *1 [lua] policy_chain.lua:199: init_worker(): policy chain execute phase: init_worker, policy: Local Policy Chain, i: 3 [debug] 19#19: *1 [lua] policy_chain.lua:199: policy chain execute phase: init_worker, policy: APIcast, i: 1 [debug] 19#19: *1 [lua] policy_chain.lua:199: init_worker(): policy chain execute phase: init_worker, policy: Metrics, i: 4 [debug] 19#19: *2 [lua] policy_loader.lua:150: loading policy: load_configuration version: builtin [debug] 19#19: *2 [lua] management.lua:177: router(): management api mode: status [debug] 19#19: *3 [lua] management.lua:177: router(): management api mode: status [debug] 19#19: *4 [lua] management.lua:177: router(): management api mode: status [debug] 19#19: *5 [lua] management.lua:177: router(): management api mode: status [debug] 19#19: *6 [lua] management.lua:177: router(): management api mode: status [debug] 19#19: *7 [lua] management.lua:177: router(): management api mode: status [debug] 19#19: *8 [lua] management.lua:177: router(): management api mode: status [debug] 19#19: *9 [lua] management.lua:177: router(): management api mode: status [debug] 19#19: *10 [lua] executor.lua:26: rewrite(): executor phase: rewrite [debug] 19#19: *10 [lua] policy_chain.lua:199: rewrite(): policy chain execute phase: rewrite, policy: Load Configuration, i: 1 [debug] 19#19: *10 [lua] env.lua:86: fetch(): env: APICAST_CONFIGURATION = nil [debug] 19#19: *10 [lua] env.lua:86: fetch(): env: THREESCALE_CONFIG_FILE = nil [debug] 19#19: *10 [lua] env.lua:86: fetch(): env: APICAST_LOAD_SERVICES_WHEN_NEEDED = nil [debug] 19#19: *10 [lua] resolver.lua:195: new(): resolver search domains: threescale-postgres.svc.cluster.local svc.cluster.local cluster.local us-east-2.compute.internal threescale-postgres.svc.cluster.local svc.cluster.local cluster.local us-east-2.compute.internal [debug] 19#19: *10 [lua] resolver.lua:326: lookup(): resolver query: system-master [debug] 19#19: *10 [lua] resolver.lua:292: search_dns(): resolver query: system-master search: query: system-master. [debug] 19#19: *10 [lua] cache.lua:122: fetch_answers(): resolver cache miss system-master. [debug] 19#19: *10 [lua] cache.lua:188: get(): resolver cache miss: system-master. [debug] 19#19: *10 [lua] dns_client.lua:43: init_resolvers(): initializing 2 nameservers [debug] 19#19: *10 [lua] dns_client.lua:56: init_resolvers(): nameserver 172.30.0.10:53 initialized [debug] 19#19: *10 [lua] dns_client.lua:56: init_resolvers(): nameserver 172.30.0.10:53 initialized [debug] 19#19: *10 [lua] dns_client.lua:68: query(): resolver query: system-master. nameserver: 172.30.0.10:53 [debug] 19#19: *10 [lua] dns_client.lua:68: query(): resolver query: system-master. nameserver: 172.30.0.10:53 [debug] 19#19: *10 [lua] resolver.lua:292: search_dns(): resolver query: system-master search: threescale-postgres.svc.cluster.local query: system-master.threescale-postgres.svc.cluster.local [debug] 19#19: *10 [lua] cache.lua:122: fetch_answers(): resolver cache miss system-master.threescale-postgres.svc.cluster.local [debug] 19#19: *10 [lua] cache.lua:188: get(): resolver cache miss: system-master.threescale-postgres.svc.cluster.local [debug] 19#19: *10 [lua] dns_client.lua:68: query(): resolver query: system-master.threescale-postgres.svc.cluster.local nameserver: 172.30.0.10:53 [debug] 19#19: *10 [lua] cache.lua:74: store(): resolver cache write system-master.threescale-postgres.svc.cluster.local with TLL 5 [debug] 19#19: *10 [lua] resolver.lua:346: lookup(): resolver query: system-master finished with 1 answers [debug] 19#19: *10 [lua] resolver.lua:386: get_servers(): query for system-master finished with 1 answers [debug] 19#19: *10 [lua] resolver.lua:326: lookup(): resolver query: 172.30.255.136 [debug] 19#19: *10 [lua] resolver.lua:331: lookup(): host is ip address: 172.30.255.136 [debug] 19#19: *10 [lua] resolver.lua:346: lookup(): resolver query: 172.30.255.136 finished with 1 answers [debug] 19#19: *10 [lua] resolver.lua:386: get_servers(): query for 172.30.255.136 finished with 1 answers [debug] 19#19: *10 [lua] http.lua:50: connect(): connected to ip:172.30.255.136 host: 172.30.255.136 port: 3000 ok: 1 err: nil [debug] 19#19: *10 [lua] proxy.lua:26: new(): connection to system-master:3000 established, reused times: 0 [debug] 19#19: *10 [lua] http.lua:633: send_request(): GET /master/api/proxy/configs/staging.json?host=pokus-3scale-apicast-staging.threescale-postgres.apps.3scale-cluster.openshift-aws.rhocf-dev.com HTTP/1.1 User-Agent: APIcast/3.7.0 (Linux; x64; env:staging) APIcast/builtin Host: system-master:3000 Connection: Keep-Alive Authorization: Basic RFhQaENCMGo6 [debug] 19#19: *10 [lua] remote_v2.lua:179: index(): index get status: 200 url: http://DXPhCB0j@system-master:3000/master/api/proxy/configs/staging.json?host=pokus-3scale-apicast-staging.threescale-postgres.apps.3scale-cluster.openshift-aws.rhocf-dev.com [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: routing version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: upstream version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: default_credentials version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: cors version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: liquid_context_debug version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: upstream version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: routing version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: routing version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: routing version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: upstream version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: default_credentials version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: cors version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: liquid_context_debug version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: upstream version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: routing version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: routing version: builtin [debug] 19#19: *10 [lua] policy_loader.lua:150: loading policy: apicast version: builtin [debug] 19#19: *10 [lua] env.lua:86: fetch(): env: APICAST_SERVICES_FILTER_BY_URL = nil [debug] 19#19: *10 [lua] env.lua:86: fetch(): env: APICAST_SERVICES_LIST = nil [debug] 19#19: *10 [lua] env.lua:86: fetch(): env: APICAST_SERVICES = nil [info] 19#19: *10 [lua] configuration_store.lua:133: store(): added service 148 configuration with hosts: pokus-3scale-apicast-production.threescale-postgres.apps.3scale-cluster.openshift-aws.rhocf-dev.com, pokus-3scale-apicast-staging.threescale-postgres.apps.3scale-cluster.openshift-aws.rhocf-dev.com ttl: 0, client: 10.130.2.104, server: _, request: "GET /pokus//hokus?user_key=405d0b9d61ca3444ecdf8870727773da HTTP/1.1", host: "pokus-3scale-apicast-staging.threescale-postgres.apps.3scale-cluster.openshift-aws.rhocf-dev.com" [debug] 19#19: *10 [lua] policy_chain.lua:199: rewrite(): policy chain execute phase: rewrite, policy: Find Service Policy, i: 2 [debug] 19#19: *10 [lua] policy_chain.lua:199: rewrite(): policy chain execute phase: rewrite, policy: Local Policy Chain, i: 3 [debug] 19#19: *10 [lua] env.lua:86: fetch(): env: APICAST_BACKEND_CACHE_HANDLER = nil [debug] 19#19: *10 [lua] cache_handler.lua:23: new(): backend cache handler: strict [debug] 19#19: *10 [lua] policy_chain.lua:199: rewrite(): policy chain execute phase: rewrite, policy: Routing policy, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: rewrite(): policy chain execute phase: rewrite, policy: APIcast, i: 2 [debug] 19#19: *10 [lua] service.lua:236: get_usage(): [mapping] service 148 has 1 rules [warn] 19#19: *10 [lua] apicast.lua:65: upstream api for the service:148 is invalid, error:Upstream cannot be null, client: 10.130.2.104, server: _, request: "GET /pokus//hokus?user_key=405d0b9d61ca3444ecdf8870727773da HTTP/1.1", host: "pokus-3scale-apicast-staging.threescale-postgres.apps.3scale-cluster.openshift-aws.rhocf-dev.com" [debug] 19#19: *10 [lua] policy_chain.lua:199: rewrite(): policy chain execute phase: rewrite, policy: Metrics, i: 4 [debug] 19#19: *10 [lua] executor.lua:26: access(): executor phase: access [debug] 19#19: *10 [lua] policy_chain.lua:199: access(): policy chain execute phase: access, policy: Load Configuration, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: access(): policy chain execute phase: access, policy: Find Service Policy, i: 2 [debug] 19#19: *10 [lua] policy_chain.lua:199: access(): policy chain execute phase: access, policy: Local Policy Chain, i: 3 [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: access, policy: Routing policy, i: 1 [debug] 19#19: *10 [lua] upstream_selector.lua:28: select(): Rule with URL https://echo-api.3scale.net:443 matched [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: access, policy: APIcast, i: 2 [info] 19#19: *10 [lua] errors.lua:41: no rules matched for service 148, client: 10.130.2.104, server: _, request: "GET /pokus//hokus?user_key=405d0b9d61ca3444ecdf8870727773da HTTP/1.1", host: "pokus-3scale-apicast-staging.threescale-postgres.apps.3scale-cluster.openshift-aws.rhocf-dev.com" [debug] 19#19: *10 [lua] executor.lua:26: header_filter(): executor phase: header_filter [debug] 19#19: *10 [lua] policy_chain.lua:199: header_filter(): policy chain execute phase: header_filter, policy: Load Configuration, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: header_filter(): policy chain execute phase: header_filter, policy: Find Service Policy, i: 2 [debug] 19#19: *10 [lua] policy_chain.lua:199: header_filter(): policy chain execute phase: header_filter, policy: Local Policy Chain, i: 3 [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: header_filter, policy: Routing policy, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: header_filter, policy: APIcast, i: 2 [debug] 19#19: *10 [lua] policy_chain.lua:199: header_filter(): policy chain execute phase: header_filter, policy: Metrics, i: 4 [debug] 19#19: *10 [lua] executor.lua:26: body_filter(): executor phase: body_filter [debug] 19#19: *10 [lua] policy_chain.lua:199: body_filter(): policy chain execute phase: body_filter, policy: Load Configuration, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: body_filter(): policy chain execute phase: body_filter, policy: Find Service Policy, i: 2 [debug] 19#19: *10 [lua] policy_chain.lua:199: body_filter(): policy chain execute phase: body_filter, policy: Local Policy Chain, i: 3 [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: body_filter, policy: Routing policy, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: body_filter, policy: APIcast, i: 2 [debug] 19#19: *10 [lua] policy_chain.lua:199: body_filter(): policy chain execute phase: body_filter, policy: Metrics, i: 4 [debug] 19#19: *10 [lua] executor.lua:26: body_filter(): executor phase: body_filter [debug] 19#19: *10 [lua] policy_chain.lua:199: body_filter(): policy chain execute phase: body_filter, policy: Load Configuration, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: body_filter(): policy chain execute phase: body_filter, policy: Find Service Policy, i: 2 [debug] 19#19: *10 [lua] policy_chain.lua:199: body_filter(): policy chain execute phase: body_filter, policy: Local Policy Chain, i: 3 [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: body_filter, policy: Routing policy, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: body_filter, policy: APIcast, i: 2 [debug] 19#19: *10 [lua] policy_chain.lua:199: body_filter(): policy chain execute phase: body_filter, policy: Metrics, i: 4 [debug] 19#19: *10 [lua] executor.lua:26: post_action(): executor phase: post_action [debug] 19#19: *10 [lua] policy_chain.lua:199: post_action(): policy chain execute phase: post_action, policy: Load Configuration, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: post_action(): policy chain execute phase: post_action, policy: Find Service Policy, i: 2 [debug] 19#19: *10 [lua] policy_chain.lua:199: post_action(): policy chain execute phase: post_action, policy: Local Policy Chain, i: 3 [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: post_action, policy: Routing policy, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: post_action, policy: APIcast, i: 2 [info] 19#19: *10 [lua] proxy.lua:340: [async] skipping after action, no cached key, client: 10.130.2.104, server: _, request: "GET /pokus//hokus?user_key=405d0b9d61ca3444ecdf8870727773da HTTP/1.1", host: "pokus-3scale-apicast-staging.threescale-postgres.apps.3scale-cluster.openshift-aws.rhocf-dev.com" [debug] 19#19: *10 [lua] policy_chain.lua:199: post_action(): policy chain execute phase: post_action, policy: Metrics, i: 4 [debug] 19#19: *10 [lua] executor.lua:26: log(): executor phase: log [debug] 19#19: *10 [lua] policy_chain.lua:199: log(): policy chain execute phase: log, policy: Load Configuration, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: log(): policy chain execute phase: log, policy: Find Service Policy, i: 2 [debug] 19#19: *10 [lua] policy_chain.lua:199: log(): policy chain execute phase: log, policy: Local Policy Chain, i: 3 [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: log, policy: Routing policy, i: 1 [debug] 19#19: *10 [lua] policy_chain.lua:199: policy chain execute phase: log, policy: APIcast, i: 2 [debug] 19#19: *10 [lua] policy_chain.lua:199: log(): policy chain execute phase: log, policy: Metrics, i: 4 [20/Nov/2019:15:49:50 +0000] pokus-3scale-apicast-staging.threescale-postgres.apps.3scale-cluster.openshift-aws.rhocf-dev.com:8080 10.130.2.104:42538 "GET /pokus//hokus?user_key=405d0b9d61ca3444ecdf8870727773da HTTP/1.1" 404 34 (0.041) 0 [info] 19#19: *10 client 10.130.2.104 closed keepalive connection (104: Connection reset by peer) [debug] 19#19: *14 [lua] management.lua:177: router(): management api mode: status [debug] 19#19: *15 [lua] management.lua:177: router(): management api mode: status [debug] 19#19: *16 [lua] management.lua:177: router(): management api mode: status