--------------------------------------------------- -- api's used to run --------------------------------------------------- cd /etc/openshift/elasticsearch/secret curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v --------------------------------------------------- -- FullRedundancy - nodeCount: 1 --------------------------------------------------- sh-4.2$ cd /etc/openshift/elasticsearch/secret sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open jaeger-service-2019-07-31 8nBisw4ZQH2KjXHgY-jYsw 1 0 4 0 3.5kb 3.5kb green open .searchguard HJnEJwdYTIy2a5qIbcz7eQ 1 0 5 0 37.6kb 37.6kb green open jaeger-span-2019-07-31 EZH40KX5Q5qETdm_q3uqDg 1 0 40 0 10kb 10kb sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty { "jaeger-span-2019-07-31": { "settings": { "index": { "mapping": { "nested_fields": { "limit": "50" } }, "number_of_shards": "1", "provided_name": "jaeger-span-2019-07-31", "mapper": { "dynamic": "false" }, "creation_date": "1564544874766", "requests": { "cache": { "enable": "true" } }, "number_of_replicas": "0", "uuid": "EZH40KX5Q5qETdm_q3uqDg", "version": { "created": "5061399" } } } } } sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v id host ip node eRU81WS8SIyZIDin5ZQpog 10.131.0.25 10.131.0.25 elasticsearch-cdm-trt9j754-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 10.131.0.25 1 84 7 1.94 1.61 1.17 mdi * elasticsearch-cdm-trt9j754-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v index shard prirep state docs store ip node .searchguard 0 p STARTED 5 37.6kb 10.131.0.25 elasticsearch-cdm-trt9j754-1 jaeger-service-2019-07-31 0 p STARTED 4 3.5kb 10.131.0.25 elasticsearch-cdm-trt9j754-1 jaeger-span-2019-07-31 0 p STARTED 40 10kb 10.131.0.25 elasticsearch-cdm-trt9j754-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent jaeger-span-2019-07-31 0 70ms empty_store done n/a n/a 10.131.0.25 elasticsearch-cdm-trt9j754-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 122ms empty_store done n/a n/a 10.131.0.25 elasticsearch-cdm-trt9j754-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 0 32ms empty_store done n/a n/a 10.131.0.25 elasticsearch-cdm-trt9j754-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% --------------------------------------------------- -- FullRedundancy - nodeCount: 3 --------------------------------------------------- sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open jaeger-service-2019-07-31 rMnTKT1kRGanL2xSpIRyxw 3 2 4 0 28.9kb 9.6kb green open .searchguard wvy7CivlSOe2MRTNPT23WQ 1 2 5 1 113.3kb 37.8kb green open jaeger-span-2019-07-31 kQZdupV-RuaP9QsQsNlRyg 3 2 40 0 53.7kb 17.9kb sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty { "jaeger-span-2019-07-31": { "settings": { "index": { "mapping": { "nested_fields": { "limit": "50" } }, "number_of_shards": "3", "provided_name": "jaeger-span-2019-07-31", "mapper": { "dynamic": "false" }, "creation_date": "1564545551604", "requests": { "cache": { "enable": "true" } }, "number_of_replicas": "2", "uuid": "kQZdupV-RuaP9QsQsNlRyg", "version": { "created": "5061399" } } } } } sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v id host ip node mnahMLiSTCqKgxxlqBpO3w 10.131.0.28 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 10.131.0.28 9 98 15 1.27 1.36 1.23 mdi * elasticsearch-cdm-zeuy37ec-1 10.130.0.169 2 94 10 0.28 0.52 0.45 mdi - elasticsearch-cdm-zeuy37ec-2 10.131.0.29 2 98 14 1.27 1.36 1.23 mdi - elasticsearch-cdm-zeuy37ec-3 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v index shard prirep state docs store ip node .searchguard 0 p STARTED 5 37.8kb 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 .searchguard 0 r STARTED 5 37.6kb 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 .searchguard 0 r STARTED 5 37.8kb 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 jaeger-service-2019-07-31 2 r STARTED 1 3.2kb 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 jaeger-service-2019-07-31 2 r STARTED 1 3.2kb 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 jaeger-service-2019-07-31 2 p STARTED 1 3.2kb 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 jaeger-service-2019-07-31 1 r STARTED 2 3.2kb 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 jaeger-service-2019-07-31 1 p STARTED 2 3.2kb 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 jaeger-service-2019-07-31 1 r STARTED 2 3.2kb 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 jaeger-service-2019-07-31 0 p STARTED 1 3.1kb 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 jaeger-service-2019-07-31 0 r STARTED 1 3.1kb 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 jaeger-service-2019-07-31 0 r STARTED 1 3.1kb 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 jaeger-span-2019-07-31 2 r STARTED 0 162b 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 jaeger-span-2019-07-31 2 r STARTED 0 162b 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 jaeger-span-2019-07-31 2 p STARTED 0 162b 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 jaeger-span-2019-07-31 1 r STARTED 33 9.7kb 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 jaeger-span-2019-07-31 1 p STARTED 33 9.7kb 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 jaeger-span-2019-07-31 1 r STARTED 33 9.7kb 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 jaeger-span-2019-07-31 0 p STARTED 7 8kb 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 jaeger-span-2019-07-31 0 r STARTED 7 8kb 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 jaeger-span-2019-07-31 0 r STARTED 7 8kb 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent jaeger-span-2019-07-31 0 52ms empty_store done n/a n/a 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-span-2019-07-31 0 219ms peer done 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 1 1 100.0% jaeger-span-2019-07-31 0 76ms peer done 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-span-2019-07-31 1 84ms peer done 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-span-2019-07-31 1 55ms empty_store done n/a n/a 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-span-2019-07-31 1 61ms peer done 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-span-2019-07-31 2 608ms peer done 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-span-2019-07-31 2 678ms peer done 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-span-2019-07-31 2 53ms empty_store done n/a n/a 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 147ms empty_store done n/a n/a 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 427ms peer done 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% .searchguard 0 593ms peer done 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 0 35ms empty_store done n/a n/a 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 0 605ms peer done 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 0 111ms peer done 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 1 130ms peer done 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 1 51ms empty_store done n/a n/a 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 1 83ms peer done 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 2 610ms peer done 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 10.130.0.169 elasticsearch-cdm-zeuy37ec-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 2 699ms peer done 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 10.131.0.29 elasticsearch-cdm-zeuy37ec-3 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 1 1 100.0% jaeger-service-2019-07-31 2 55ms empty_store done n/a n/a 10.131.0.28 elasticsearch-cdm-zeuy37ec-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% --------------------------------------------------- -- MultipleRedundancy - nodeCount: 3 --------------------------------------------------- sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open .searchguard O-hw58i_R7G7hj4ivRY3gg 1 1 5 0 75.3kb 37.6kb green open jaeger-service-2019-07-31 Jga6QyqbQQ26AHf3gxSmzA 3 1 4 0 19.2kb 9.6kb green open jaeger-span-2019-07-31 EU3UrCYAQAm72Rd_9EGpFQ 3 1 40 0 21kb 10.5kb sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty { "jaeger-span-2019-07-31": { "settings": { "index": { "mapping": { "nested_fields": { "limit": "50" } }, "number_of_shards": "3", "provided_name": "jaeger-span-2019-07-31", "mapper": { "dynamic": "false" }, "creation_date": "1564546422002", "requests": { "cache": { "enable": "true" } }, "number_of_replicas": "1", "uuid": "EU3UrCYAQAm72Rd_9EGpFQ", "version": { "created": "5061399" } } } } } sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v id host ip node oGeIbTrFQ0WiVBJTerIUUA 10.131.0.39 10.131.0.39 elasticsearch-cdm-8576jubm-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 10.131.0.39 8 83 15 0.77 1.68 1.62 mdi * elasticsearch-cdm-8576jubm-1 10.130.0.172 15 95 10 0.25 0.51 0.54 mdi - elasticsearch-cdm-8576jubm-2 10.131.0.40 14 83 14 0.77 1.68 1.62 mdi - elasticsearch-cdm-8576jubm-3 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v index shard prirep state docs store ip node jaeger-span-2019-07-31 1 p STARTED 0 162b 10.130.0.172 elasticsearch-cdm-8576jubm-2 jaeger-span-2019-07-31 1 r STARTED 0 162b 10.131.0.39 elasticsearch-cdm-8576jubm-1 jaeger-span-2019-07-31 2 p STARTED 0 162b 10.131.0.40 elasticsearch-cdm-8576jubm-3 jaeger-span-2019-07-31 2 r STARTED 0 162b 10.131.0.39 elasticsearch-cdm-8576jubm-1 jaeger-span-2019-07-31 0 r STARTED 40 10.2kb 10.130.0.172 elasticsearch-cdm-8576jubm-2 jaeger-span-2019-07-31 0 p STARTED 40 10.2kb 10.131.0.40 elasticsearch-cdm-8576jubm-3 .searchguard 0 p STARTED 5 37.6kb 10.130.0.172 elasticsearch-cdm-8576jubm-2 .searchguard 0 r STARTED 5 37.6kb 10.131.0.39 elasticsearch-cdm-8576jubm-1 jaeger-service-2019-07-31 1 p STARTED 2 3.2kb 10.130.0.172 elasticsearch-cdm-8576jubm-2 jaeger-service-2019-07-31 1 r STARTED 2 3.2kb 10.131.0.40 elasticsearch-cdm-8576jubm-3 jaeger-service-2019-07-31 2 r STARTED 1 3.2kb 10.131.0.40 elasticsearch-cdm-8576jubm-3 jaeger-service-2019-07-31 2 p STARTED 1 3.2kb 10.131.0.39 elasticsearch-cdm-8576jubm-1 jaeger-service-2019-07-31 0 r STARTED 1 3.1kb 10.130.0.172 elasticsearch-cdm-8576jubm-2 jaeger-service-2019-07-31 0 p STARTED 1 3.1kb 10.131.0.39 elasticsearch-cdm-8576jubm-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent jaeger-span-2019-07-31 0 643ms peer done 10.131.0.39 elasticsearch-cdm-8576jubm-1 10.130.0.172 elasticsearch-cdm-8576jubm-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 4 4 100.0% jaeger-span-2019-07-31 0 674ms peer done 10.131.0.39 elasticsearch-cdm-8576jubm-1 10.131.0.40 elasticsearch-cdm-8576jubm-3 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 4 4 100.0% jaeger-span-2019-07-31 1 59ms empty_store done n/a n/a 10.130.0.172 elasticsearch-cdm-8576jubm-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-span-2019-07-31 1 66ms peer done 10.130.0.172 elasticsearch-cdm-8576jubm-2 10.131.0.39 elasticsearch-cdm-8576jubm-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-span-2019-07-31 2 68ms empty_store done n/a n/a 10.131.0.40 elasticsearch-cdm-8576jubm-3 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-span-2019-07-31 2 86ms peer done 10.131.0.40 elasticsearch-cdm-8576jubm-3 10.131.0.39 elasticsearch-cdm-8576jubm-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% .searchguard 0 178ms empty_store done n/a n/a 10.130.0.172 elasticsearch-cdm-8576jubm-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 514ms peer done 10.130.0.172 elasticsearch-cdm-8576jubm-2 10.131.0.39 elasticsearch-cdm-8576jubm-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 0 144ms peer done 10.131.0.40 elasticsearch-cdm-8576jubm-3 10.130.0.172 elasticsearch-cdm-8576jubm-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 0 122ms peer done 10.131.0.40 elasticsearch-cdm-8576jubm-3 10.131.0.39 elasticsearch-cdm-8576jubm-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 1 1 100.0% jaeger-service-2019-07-31 1 124ms empty_store done n/a n/a 10.130.0.172 elasticsearch-cdm-8576jubm-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 1 699ms peer done 10.130.0.172 elasticsearch-cdm-8576jubm-2 10.131.0.40 elasticsearch-cdm-8576jubm-3 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 1 1 100.0% jaeger-service-2019-07-31 2 696ms peer done 10.131.0.39 elasticsearch-cdm-8576jubm-1 10.131.0.40 elasticsearch-cdm-8576jubm-3 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 1 1 100.0% jaeger-service-2019-07-31 2 39ms empty_store done n/a n/a 10.131.0.39 elasticsearch-cdm-8576jubm-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% sh-4.2$ --------------------------------------------------- -- MultipleRedundancy - nodeCount: 2 --------------------------------------------------- sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open jaeger-span-2019-07-31 7AcagIVgTduX15Vxfebiuw 2 0 40 0 17.2kb 17.2kb green open .searchguard gDlqXJKlTnGes52ODHuDAg 1 0 5 2 37.8kb 37.8kb green open jaeger-service-2019-07-31 BC6WWMUrSbiTustJAWMMAA 2 0 4 0 6.5kb 6.5kb sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty { "jaeger-span-2019-07-31": { "settings": { "index": { "mapping": { "nested_fields": { "limit": "50" } }, "number_of_shards": "2", "provided_name": "jaeger-span-2019-07-31", "mapper": { "dynamic": "false" }, "creation_date": "1564546863991", "requests": { "cache": { "enable": "true" } }, "number_of_replicas": "0", "uuid": "7AcagIVgTduX15Vxfebiuw", "version": { "created": "5061399" } } } } } sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v id host ip node ZdHUoS0jRraAoRcQT9vDjQ 10.130.0.173 10.130.0.173 elasticsearch-cdm-ky95fsy2-2 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 10.130.0.173 11 95 13 0.15 0.27 0.41 mdi * elasticsearch-cdm-ky95fsy2-2 10.131.0.44 9 69 13 1.84 1.67 1.55 mdi - elasticsearch-cdm-ky95fsy2-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v index shard prirep state docs store ip node .searchguard 0 p STARTED 5 37.8kb 10.130.0.173 elasticsearch-cdm-ky95fsy2-2 jaeger-span-2019-07-31 1 p STARTED 13 7.7kb 10.130.0.173 elasticsearch-cdm-ky95fsy2-2 jaeger-span-2019-07-31 0 p STARTED 27 9.5kb 10.131.0.44 elasticsearch-cdm-ky95fsy2-1 jaeger-service-2019-07-31 1 p STARTED 3 3.4kb 10.130.0.173 elasticsearch-cdm-ky95fsy2-2 jaeger-service-2019-07-31 0 p STARTED 1 3.1kb 10.131.0.44 elasticsearch-cdm-ky95fsy2-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent jaeger-span-2019-07-31 0 44ms empty_store done n/a n/a 10.131.0.44 elasticsearch-cdm-ky95fsy2-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-span-2019-07-31 1 53ms empty_store done n/a n/a 10.130.0.173 elasticsearch-cdm-ky95fsy2-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 140ms empty_store done n/a n/a 10.130.0.173 elasticsearch-cdm-ky95fsy2-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 0 33ms empty_store done n/a n/a 10.131.0.44 elasticsearch-cdm-ky95fsy2-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 1 34ms empty_store done n/a n/a 10.130.0.173 elasticsearch-cdm-ky95fsy2-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% --------------------------------------------------- -- MultipleRedundancy - nodeCount: 1 --------------------------------------------------- sh-4.2$ cd /etc/openshift/elasticsearch/secret sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open jaeger-service-2019-07-31 GZE5sIp1QqmQCp0QUxZmCg 1 0 4 0 3.5kb 3.5kb green open jaeger-span-2019-07-31 3RZuhhKLSC2st_sYgkztTA 1 0 40 0 10.2kb 10.2kb green open .searchguard RHxWrRqqTnSSMIJTsRuvAA 1 0 5 0 37.6kb 37.6kb sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty { "jaeger-span-2019-07-31": { "settings": { "index": { "mapping": { "nested_fields": { "limit": "50" } }, "number_of_shards": "1", "provided_name": "jaeger-span-2019-07-31", "mapper": { "dynamic": "false" }, "creation_date": "1564547231460", "requests": { "cache": { "enable": "true" } }, "number_of_replicas": "0", "uuid": "3RZuhhKLSC2st_sYgkztTA", "version": { "created": "5061399" } } } } } sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v id host ip node LrQHYpyLRb6XzbgNx4gsmw 10.131.0.46 10.131.0.46 elasticsearch-cdm-4uijjhib-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 10.131.0.46 7 69 8 0.45 0.82 1.19 mdi * elasticsearch-cdm-4uijjhib-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v index shard prirep state docs store ip node .searchguard 0 p STARTED 5 37.6kb 10.131.0.46 elasticsearch-cdm-4uijjhib-1 jaeger-span-2019-07-31 0 p STARTED 40 10.2kb 10.131.0.46 elasticsearch-cdm-4uijjhib-1 jaeger-service-2019-07-31 0 p STARTED 4 3.5kb 10.131.0.46 elasticsearch-cdm-4uijjhib-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent jaeger-span-2019-07-31 0 49ms empty_store done n/a n/a 10.131.0.46 elasticsearch-cdm-4uijjhib-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 130ms empty_store done n/a n/a 10.131.0.46 elasticsearch-cdm-4uijjhib-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 0 54ms empty_store done n/a n/a 10.131.0.46 elasticsearch-cdm-4uijjhib-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% --------------------------------------------------- -- SingleRedundancy - nodeCount: 3 --------------------------------------------------- sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open jaeger-span-2019-07-31 n9sp5J9yRsKC0Ey4TTrBeQ 3 1 40 0 20.8kb 10.4kb green open jaeger-service-2019-07-31 vDLwgiOLRaa8aMFx7vbXxw 3 1 4 0 22.1kb 9.6kb green open .searchguard Y8_N02q-TT2gQ5sO42cn4A 1 1 5 0 75.5kb 37.6kb sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty { "jaeger-span-2019-07-31": { "settings": { "index": { "mapping": { "nested_fields": { "limit": "50" } }, "number_of_shards": "3", "provided_name": "jaeger-span-2019-07-31", "mapper": { "dynamic": "false" }, "creation_date": "1564554848451", "requests": { "cache": { "enable": "true" } }, "number_of_replicas": "1", "uuid": "n9sp5J9yRsKC0Ey4TTrBeQ", "version": { "created": "5061399" } } } } } sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v id host ip node 1T8stzFKTuOBjax_RJizCQ 10.130.0.175 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 10.130.0.175 3 95 12 0.32 0.31 0.29 mdi * elasticsearch-cdm-awxp3aqs-2 10.131.0.54 12 85 20 0.32 0.56 0.33 mdi - elasticsearch-cdm-awxp3aqs-1 10.131.0.55 11 85 17 0.32 0.56 0.33 mdi - elasticsearch-cdm-awxp3aqs-3 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v index shard prirep state docs store ip node jaeger-span-2019-07-31 1 p STARTED 0 162b 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 jaeger-span-2019-07-31 1 r STARTED 0 162b 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 jaeger-span-2019-07-31 2 p STARTED 0 162b 10.131.0.54 elasticsearch-cdm-awxp3aqs-1 jaeger-span-2019-07-31 2 r STARTED 0 162b 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 jaeger-span-2019-07-31 0 r STARTED 40 10.1kb 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 jaeger-span-2019-07-31 0 p STARTED 40 10.1kb 10.131.0.54 elasticsearch-cdm-awxp3aqs-1 .searchguard 0 p STARTED 5 37.6kb 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 .searchguard 0 r STARTED 5 37.8kb 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 jaeger-service-2019-07-31 1 p STARTED 2 3.2kb 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 jaeger-service-2019-07-31 1 r STARTED 2 6.1kb 10.131.0.54 elasticsearch-cdm-awxp3aqs-1 jaeger-service-2019-07-31 2 r STARTED 1 3.2kb 10.131.0.54 elasticsearch-cdm-awxp3aqs-1 jaeger-service-2019-07-31 2 p STARTED 1 3.2kb 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 jaeger-service-2019-07-31 0 r STARTED 1 3.1kb 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 jaeger-service-2019-07-31 0 p STARTED 1 3.1kb 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent jaeger-span-2019-07-31 0 710ms peer done 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 4 4 100.0% jaeger-span-2019-07-31 0 679ms peer done 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 10.131.0.54 elasticsearch-cdm-awxp3aqs-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 4 4 100.0% jaeger-span-2019-07-31 1 78ms empty_store done n/a n/a 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-span-2019-07-31 1 149ms peer done 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-span-2019-07-31 2 82ms empty_store done n/a n/a 10.131.0.54 elasticsearch-cdm-awxp3aqs-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-span-2019-07-31 2 200ms peer done 10.131.0.54 elasticsearch-cdm-awxp3aqs-1 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% .searchguard 0 204ms empty_store done n/a n/a 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 330ms peer done 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 0 667ms peer done 10.131.0.54 elasticsearch-cdm-awxp3aqs-1 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 1 1 100.0% jaeger-service-2019-07-31 0 74ms peer done 10.131.0.54 elasticsearch-cdm-awxp3aqs-1 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 1 1 100.0% jaeger-service-2019-07-31 1 50ms empty_store done n/a n/a 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 1 125ms peer done 10.131.0.55 elasticsearch-cdm-awxp3aqs-3 10.131.0.54 elasticsearch-cdm-awxp3aqs-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 2 673ms peer done 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 10.131.0.54 elasticsearch-cdm-awxp3aqs-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 1 1 100.0% jaeger-service-2019-07-31 2 28ms empty_store done n/a n/a 10.130.0.175 elasticsearch-cdm-awxp3aqs-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% --------------------------------------------------- -- SingleRedundancy - nodeCount: 2 --------------------------------------------------- sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open jaeger-service-2019-07-31 dfUEhA9IQ3yZbBjryMdCCw 2 1 4 0 13.1kb 6.5kb green open jaeger-span-2019-07-31 DTVsaWD9SV2MOtF3C1sDmw 2 1 40 0 34.7kb 17.3kb green open .searchguard 4UTutwFcRyu_7TUJAK2sSA 1 1 5 0 75.3kb 37.6kb sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty { "jaeger-span-2019-07-31": { "settings": { "index": { "mapping": { "nested_fields": { "limit": "50" } }, "number_of_shards": "2", "provided_name": "jaeger-span-2019-07-31", "mapper": { "dynamic": "false" }, "creation_date": "1564555737328", "requests": { "cache": { "enable": "true" } }, "number_of_replicas": "1", "uuid": "DTVsaWD9SV2MOtF3C1sDmw", "version": { "created": "5061399" } } } } } sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v id host ip node AXwjaJHuSoKJx1BIJyJW7w 10.131.0.58 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 10.131.0.58 11 71 13 0.36 0.41 0.32 mdi * elasticsearch-cdm-gjjsxfhk-1 10.130.0.176 9 96 13 0.08 0.15 0.18 mdi - elasticsearch-cdm-gjjsxfhk-2 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v index shard prirep state docs store ip node jaeger-span-2019-07-31 1 r STARTED 13 7.5kb 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 jaeger-span-2019-07-31 1 p STARTED 13 7.5kb 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 jaeger-span-2019-07-31 0 p STARTED 27 9.8kb 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 jaeger-span-2019-07-31 0 r STARTED 27 9.8kb 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 jaeger-service-2019-07-31 1 r STARTED 3 3.4kb 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 jaeger-service-2019-07-31 1 p STARTED 3 3.4kb 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 jaeger-service-2019-07-31 0 p STARTED 1 3.1kb 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 jaeger-service-2019-07-31 0 r STARTED 1 3.1kb 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 .searchguard 0 p STARTED 5 37.6kb 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 .searchguard 0 r STARTED 5 37.6kb 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent jaeger-span-2019-07-31 0 39ms empty_store done n/a n/a 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-span-2019-07-31 0 61ms peer done 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-span-2019-07-31 1 629ms peer done 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 1 1 100.0% jaeger-span-2019-07-31 1 42ms empty_store done n/a n/a 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 139ms empty_store done n/a n/a 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 312ms peer done 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 0 28ms empty_store done n/a n/a 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 0 110ms peer done 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 0 0 100.0% jaeger-service-2019-07-31 1 647ms peer done 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 10.130.0.176 elasticsearch-cdm-gjjsxfhk-2 n/a n/a 1 1 100.0% 1 162 162 100.0% 162 3 3 100.0% jaeger-service-2019-07-31 1 105ms empty_store done n/a n/a 10.131.0.58 elasticsearch-cdm-gjjsxfhk-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% --------------------------------------------------- -- SingleRedundancy - nodeCount: 1 and 4 --------------------------------------------------- FAILED, es cluster not created. I do not see and error message on es-operator/jaeger-operator --------------------------------------------------- -- ZeroRedundancy - nodeCount: 1 --------------------------------------------------- sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open jaeger-service-2019-07-31 Sm1SRExtRoykcR8IddmbLw 1 0 4 0 3.5kb 3.5kb green open jaeger-span-2019-07-31 Ft9vyK0ERbmAbCQ8ny35pQ 1 0 40 0 10.1kb 10.1kb green open .searchguard VXBNLOAcRXqCTWU4vTv_Ow 1 0 5 0 37.6kb 37.6kb sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty { "jaeger-span-2019-07-31": { "settings": { "index": { "mapping": { "nested_fields": { "limit": "50" } }, "number_of_shards": "1", "provided_name": "jaeger-span-2019-07-31", "mapper": { "dynamic": "false" }, "creation_date": "1564556857830", "requests": { "cache": { "enable": "true" } }, "number_of_replicas": "0", "uuid": "Ft9vyK0ERbmAbCQ8ny35pQ", "version": { "created": "5061399" } } } } } sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v id host ip node JhtLxGO_SPaBmSptoT_UgA 10.131.0.66 10.131.0.66 elasticsearch-cdm-7tc76ilp-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 10.131.0.66 5 70 18 1.49 0.79 0.51 mdi * elasticsearch-cdm-7tc76ilp-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v index shard prirep state docs store ip node .searchguard 0 p STARTED 5 37.6kb 10.131.0.66 elasticsearch-cdm-7tc76ilp-1 jaeger-span-2019-07-31 0 p STARTED 40 10.1kb 10.131.0.66 elasticsearch-cdm-7tc76ilp-1 jaeger-service-2019-07-31 0 p STARTED 4 3.5kb 10.131.0.66 elasticsearch-cdm-7tc76ilp-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent jaeger-span-2019-07-31 0 45ms empty_store done n/a n/a 10.131.0.66 elasticsearch-cdm-7tc76ilp-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 153ms empty_store done n/a n/a 10.131.0.66 elasticsearch-cdm-7tc76ilp-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 0 31ms empty_store done n/a n/a 10.131.0.66 elasticsearch-cdm-7tc76ilp-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% --------------------------------------------------- -- ZeroRedundancy - nodeCount: 2 --------------------------------------------------- sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open jaeger-span-2019-07-31 fip4PiT2TZSupwi0O1RGmA 2 0 40 0 10.3kb 10.3kb green open .searchguard JwZqb3-9Q3C6W11UHmYUJg 1 0 5 0 37.6kb 37.6kb green open jaeger-service-2019-07-31 YEvzKSS7Q3GNCoTTotblBg 2 0 4 0 6.5kb 6.5kb sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty { "jaeger-span-2019-07-31" : { "settings" : { "index" : { "mapping" : { "nested_fields" : { "limit" : "50" } }, "number_of_shards" : "2", "provided_name" : "jaeger-span-2019-07-31", "mapper" : { "dynamic" : "false" }, "creation_date" : "1564559043901", "requests" : { "cache" : { "enable" : "true" } }, "number_of_replicas" : "0", "uuid" : "fip4PiT2TZSupwi0O1RGmA", "version" : { "created" : "5061399" } } } } } sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v id host ip node NfxqYnLORRu_-i3zsUseVg 10.130.0.178 10.130.0.178 elasticsearch-cdm-o7t08ifq-2 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 10.130.0.178 8 95 7 2.71 1.53 0.79 mdi * elasticsearch-cdm-o7t08ifq-2 10.131.0.71 2 70 8 0.89 0.82 0.74 mdi - elasticsearch-cdm-o7t08ifq-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v index shard prirep state docs store ip node jaeger-service-2019-07-31 1 p STARTED 3 3.4kb 10.130.0.178 elasticsearch-cdm-o7t08ifq-2 jaeger-service-2019-07-31 0 p STARTED 1 3.1kb 10.131.0.71 elasticsearch-cdm-o7t08ifq-1 jaeger-span-2019-07-31 1 p STARTED 0 162b 10.130.0.178 elasticsearch-cdm-o7t08ifq-2 jaeger-span-2019-07-31 0 p STARTED 40 10.2kb 10.131.0.71 elasticsearch-cdm-o7t08ifq-1 .searchguard 0 p STARTED 5 37.6kb 10.130.0.178 elasticsearch-cdm-o7t08ifq-2 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent jaeger-span-2019-07-31 0 41ms empty_store done n/a n/a 10.131.0.71 elasticsearch-cdm-o7t08ifq-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-span-2019-07-31 1 37ms empty_store done n/a n/a 10.130.0.178 elasticsearch-cdm-o7t08ifq-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 146ms empty_store done n/a n/a 10.130.0.178 elasticsearch-cdm-o7t08ifq-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 0 36ms empty_store done n/a n/a 10.131.0.71 elasticsearch-cdm-o7t08ifq-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 1 37ms empty_store done n/a n/a 10.130.0.178 elasticsearch-cdm-o7t08ifq-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% --------------------------------------------------- -- ZeroRedundancy - nodeCount: 3 --------------------------------------------------- sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open jaeger-span-2019-07-31 PU3pYR-_TLmr4aMIGriqVA 3 0 40 0 17.8kb 17.8kb green open .searchguard nn_UQF39RMSIz4eG-wyRdw 1 0 5 0 37.6kb 37.6kb green open jaeger-service-2019-07-31 TVVeANxTSxKpnVHWpfn8mA 3 0 4 0 9.6kb 9.6kb sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty { "jaeger-span-2019-07-31" : { "settings" : { "index" : { "mapping" : { "nested_fields" : { "limit" : "50" } }, "number_of_shards" : "3", "provided_name" : "jaeger-span-2019-07-31", "mapper" : { "dynamic" : "false" }, "creation_date" : "1564559663330", "requests" : { "cache" : { "enable" : "true" } }, "number_of_replicas" : "0", "uuid" : "PU3pYR-_TLmr4aMIGriqVA", "version" : { "created" : "5061399" } } } } } sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v id host ip node 7a7HfFaSQfKeYy81g4NoSg 10.130.0.179 10.130.0.179 elasticsearch-cdm-yzpo0vkr-2 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 10.131.0.77 4 85 13 0.51 0.86 0.85 mdi - elasticsearch-cdm-yzpo0vkr-3 10.131.0.76 3 85 14 0.51 0.86 0.85 mdi - elasticsearch-cdm-yzpo0vkr-1 10.130.0.179 9 96 9 0.06 0.50 0.67 mdi * elasticsearch-cdm-yzpo0vkr-2 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v index shard prirep state docs store ip node jaeger-span-2019-07-31 2 p STARTED 7 8kb 10.131.0.77 elasticsearch-cdm-yzpo0vkr-3 jaeger-span-2019-07-31 1 p STARTED 0 162b 10.130.0.179 elasticsearch-cdm-yzpo0vkr-2 jaeger-span-2019-07-31 0 p STARTED 33 9.6kb 10.131.0.76 elasticsearch-cdm-yzpo0vkr-1 .searchguard 0 p STARTED 5 37.6kb 10.131.0.77 elasticsearch-cdm-yzpo0vkr-3 jaeger-service-2019-07-31 2 p STARTED 1 3.2kb 10.131.0.77 elasticsearch-cdm-yzpo0vkr-3 jaeger-service-2019-07-31 1 p STARTED 2 3.2kb 10.130.0.179 elasticsearch-cdm-yzpo0vkr-2 jaeger-service-2019-07-31 0 p STARTED 1 3.1kb 10.131.0.76 elasticsearch-cdm-yzpo0vkr-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent jaeger-span-2019-07-31 0 72ms empty_store done n/a n/a 10.131.0.76 elasticsearch-cdm-yzpo0vkr-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-span-2019-07-31 1 69ms empty_store done n/a n/a 10.130.0.179 elasticsearch-cdm-yzpo0vkr-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-span-2019-07-31 2 69ms empty_store done n/a n/a 10.131.0.77 elasticsearch-cdm-yzpo0vkr-3 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 267ms empty_store done n/a n/a 10.131.0.77 elasticsearch-cdm-yzpo0vkr-3 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 0 47ms empty_store done n/a n/a 10.131.0.76 elasticsearch-cdm-yzpo0vkr-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 1 32ms empty_store done n/a n/a 10.130.0.179 elasticsearch-cdm-yzpo0vkr-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 2 50ms empty_store done n/a n/a 10.131.0.77 elasticsearch-cdm-yzpo0vkr-3 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% --------------------------------------------------- -- ZeroRedundancy - nodeCount: 4 --------------------------------------------------- sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://elasticsearch:9200/_cat/indices?v health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open .searchguard YAsiC_hWSt2UFgQPCzcdtw 1 0 5 0 37.6kb 37.6kb green open jaeger-span-2019-07-31 g-yteC8MRmism321CbVXbQ 1 0 40 0 10.1kb 10.1kb green open jaeger-service-2019-07-31 CZ8S4ZnNRYWp1zRVPUSknA 1 0 4 0 3.5kb 3.5kb sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/jaeger-span-2019-07-31/_settings?pretty { "jaeger-span-2019-07-31" : { "settings" : { "index" : { "mapping" : { "nested_fields" : { "limit" : "50" } }, "number_of_shards" : "1", "provided_name" : "jaeger-span-2019-07-31", "mapper" : { "dynamic" : "false" }, "creation_date" : "1564560017343", "requests" : { "cache" : { "enable" : "true" } }, "number_of_replicas" : "0", "uuid" : "g-yteC8MRmism321CbVXbQ", "version" : { "created" : "5061399" } } } } } sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/master?v id host ip node VVUWT-3gSVOTv-iIZC00LQ 10.130.0.180 10.130.0.180 elasticsearch-m-9cmcyxou sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/nodes?v ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name 10.131.0.84 4 97 22 1.04 1.24 0.99 mi - elasticsearch-m-9cmcyxou 10.131.0.83 13 97 28 1.04 1.24 0.99 di - elasticsearch-cd-owzj2cx8-1 10.130.0.180 13 96 13 0.37 0.45 0.59 mi * elasticsearch-m-9cmcyxou 10.131.0.82 5 97 31 1.04 1.24 0.99 mi - elasticsearch-m-9cmcyxou sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/shards?v index shard prirep state docs store ip node jaeger-span-2019-07-31 0 p STARTED 40 10.1kb 10.131.0.83 elasticsearch-cd-owzj2cx8-1 .searchguard 0 p STARTED 5 37.6kb 10.131.0.83 elasticsearch-cd-owzj2cx8-1 jaeger-service-2019-07-31 0 p STARTED 4 3.5kb 10.131.0.83 elasticsearch-cd-owzj2cx8-1 sh-4.2$ curl --cacert ./admin-ca --key ./admin-key --cert ./admin-cert https://localhost:9200/_cat/recovery?v index shard time type stage source_host source_node target_host target_node repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total translog_ops translog_ops_recovered translog_ops_percent jaeger-span-2019-07-31 0 52ms empty_store done n/a n/a 10.131.0.83 elasticsearch-cd-owzj2cx8-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% .searchguard 0 329ms empty_store done n/a n/a 10.131.0.83 elasticsearch-cd-owzj2cx8-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% jaeger-service-2019-07-31 0 37ms empty_store done n/a n/a 10.131.0.83 elasticsearch-cd-owzj2cx8-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0% ----------------------------- END ----------------------------------