[Kong 1.1.1] [k8s][IPv6][Cassandra db] How to configure Kong 1.1.1 to contact Cassandra via IPv6

[angrosz]

@thibaultcha
I have Kong 1.1.1 installed in IPv6 only k8s cluster in a namespace ckng. I have also Cassandra installed (cassandra-3.11.6) in a separate namespace ccas. I cannot start kong because I'm receiving the following error:

2020/04/05 12:45:14 [error] could not resolve Cassandra contact point 'ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local': failed to create a resolver: failed to set peer name: host or service not provided, or not known
Error: could not resolve any of the provided Cassandra contact points (cassandra_contact_points = 'ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local')

Debug version:

bash-4.2$ kong start --vv
2020/04/05 12:45:25 [verbose] Kong: 1.1.1
2020/04/05 12:45:25 [debug] ngx_lua: 10013
2020/04/05 12:45:25 [debug] nginx: 1013006
2020/04/05 12:45:25 [debug] Lua: LuaJIT 2.1.0-beta3
2020/04/05 12:45:25 [verbose] no config file found at /etc/kong/kong.conf
2020/04/05 12:45:25 [verbose] no config file, skip loading
2020/04/05 12:45:25 [debug] reading environment variables
2020/04/05 12:45:25 [debug] KONG_CASSANDRA_USERNAME ENV found with "suadmincass"
2020/04/05 12:45:25 [debug] KONG_CASSANDRA_PASSWORD ENV found with "******"
2020/04/05 12:45:25 [debug] KONG_DB_UPDATE_PROPAGATION ENV found with "1"
2020/04/05 12:45:25 [debug] KONG_DNS_ORDER ENV found with "LAST,SRV,A,CNAME,AAAA"
2020/04/05 12:45:25 [debug] KONG_DATABASE ENV found with "cassandra"
2020/04/05 12:45:25 [debug] KONG_LARGE_CLIENT_HEADER_BUFFERS_NUMBER ENV found with "4"
2020/04/05 12:45:25 [debug] KONG_CLIENT_HEADER_BUFFER_SIZE ENV found with "1k"
2020/04/05 12:45:25 [debug] KONG_CASSANDRA_SCHEMA_CONSENSUS_TIMEOUT ENV found with "180000"
2020/04/05 12:45:25 [debug] KONG_CASSANDRA_CONTACT_POINTS ENV found with "ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local"
2020/04/05 12:45:25 [debug] KONG_LARGE_CLIENT_HEADER_BUFFERS_SIZE ENV found with "8k"
2020/04/05 12:45:25 [debug] KONG_CASSANDRA_REPL_FACTOR ENV found with "1"
2020/04/05 12:45:25 [debug] KONG_NGINX_DAEMON ENV found with "off"
2020/04/05 12:45:25 [debug] KONG_ANONYMOUS_REPORTS ENV found with "off"
2020/04/05 12:45:25 [debug] KONG_CASSANDRA_CONSISTENCY ENV found with "QUORUM"
2020/04/05 12:45:25 [debug] admin_access_log = "logs/admin_access.log"
2020/04/05 12:45:25 [debug] admin_api_spec_listen = {"0.0.0.0:8080","0.0.0.0:9443 ssl"}
2020/04/05 12:45:25 [debug] admin_error_log = "logs/error.log"
2020/04/05 12:45:25 [debug] admin_listen = {"0.0.0.0:8001","0.0.0.0:8444 ssl"}
2020/04/05 12:45:25 [debug] anonymous_reports = false
2020/04/05 12:45:25 [debug] cache_negative_ttl_default = 60
2020/04/05 12:45:25 [debug] cache_negative_ttl_max = 300
2020/04/05 12:45:25 [debug] cache_negative_ttl_min = 1
2020/04/05 12:45:25 [debug] cache_ttl_default = 3600
2020/04/05 12:45:25 [debug] cache_ttl_max = 3600
2020/04/05 12:45:25 [debug] cache_ttl_min = 60
2020/04/05 12:45:25 [debug] cassandra_cluster_refresh_interval = 15
2020/04/05 12:45:25 [debug] cassandra_consistency = "QUORUM"
2020/04/05 12:45:25 [debug] cassandra_contact_points = {"ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local"}
2020/04/05 12:45:25 [debug] cassandra_data_centers = {"dc1:2","dc2:3"}
2020/04/05 12:45:25 [debug] cassandra_keyspace = "kong"
2020/04/05 12:45:25 [debug] cassandra_lb_policy = "RequestRoundRobin"
2020/04/05 12:45:25 [debug] cassandra_password = "******"
2020/04/05 12:45:25 [debug] cassandra_port = 9042
2020/04/05 12:45:25 [debug] cassandra_repl_factor = 1
2020/04/05 12:45:25 [debug] cassandra_repl_strategy = "SimpleStrategy"
2020/04/05 12:45:25 [debug] cassandra_schema_consensus_timeout = 180000
2020/04/05 12:45:25 [debug] cassandra_ssl = false
2020/04/05 12:45:25 [debug] cassandra_ssl_verify = false
2020/04/05 12:45:25 [debug] cassandra_timeout = 5000
2020/04/05 12:45:25 [debug] cassandra_username = "suadmincass"
2020/04/05 12:45:25 [debug] client_body_buffer_size = "8k"
2020/04/05 12:45:25 [debug] client_header_buffer_size = "1k"
2020/04/05 12:45:25 [debug] client_max_body_size = "0"
2020/04/05 12:45:25 [debug] client_ssl = false
2020/04/05 12:45:25 [debug] database = "cassandra"
2020/04/05 12:45:25 [debug] db_cache_ttl = 0
2020/04/05 12:45:25 [debug] db_resurrect_ttl = 30
2020/04/05 12:45:25 [debug] db_update_frequency = 5
2020/04/05 12:45:25 [debug] db_update_propagation = 1
2020/04/05 12:45:25 [debug] dns_error_ttl = 1
2020/04/05 12:45:25 [debug] dns_hostsfile = "/etc/hosts"
2020/04/05 12:45:25 [debug] dns_no_sync = false
2020/04/05 12:45:25 [debug] dns_not_found_ttl = 30
2020/04/05 12:45:25 [debug] dns_order = {"LAST","SRV","A","CNAME","AAAA"}
2020/04/05 12:45:25 [debug] dns_resolver = {}
2020/04/05 12:45:25 [debug] dns_stale_ttl = 4
2020/04/05 12:45:25 [debug] error_default_type = "text/plain"
2020/04/05 12:45:25 [debug] headers = {"server_tokens","latency_tokens"}
2020/04/05 12:45:25 [debug] large_client_header_buffers_number = 4
2020/04/05 12:45:25 [debug] large_client_header_buffers_size = "8k"
2020/04/05 12:45:25 [debug] log_anonymize = "OFF"
2020/04/05 12:45:25 [debug] log_level = "notice"
2020/04/05 12:45:25 [debug] lru_cache_size = 500000
2020/04/05 12:45:25 [debug] lua_package_cpath = ""
2020/04/05 12:45:25 [debug] lua_package_path = "./?.lua;./?/init.lua;"
2020/04/05 12:45:25 [debug] lua_socket_pool_size = 30
2020/04/05 12:45:25 [debug] lua_ssl_verify_depth = 1
2020/04/05 12:45:25 [debug] maria_database = "kong"
2020/04/05 12:45:25 [debug] maria_host = "127.0.0.1"
2020/04/05 12:45:25 [debug] maria_password = "******"
2020/04/05 12:45:25 [debug] maria_port = 3306
2020/04/05 12:45:25 [debug] maria_socket_keepalive_timeout = 60000
2020/04/05 12:45:25 [debug] maria_socket_pool_size = 20
2020/04/05 12:45:25 [debug] maria_ssl = false
2020/04/05 12:45:25 [debug] maria_ssl_verify = false
2020/04/05 12:45:25 [debug] maria_timeout = 10000
2020/04/05 12:45:25 [debug] maria_user = "kong"
2020/04/05 12:45:25 [debug] mem_cache_size = "128m"
2020/04/05 12:45:25 [debug] nginx_admin_directives = {}
2020/04/05 12:45:25 [debug] nginx_daemon = "off"
2020/04/05 12:45:25 [debug] nginx_http_directives = {}
2020/04/05 12:45:25 [debug] nginx_optimizations = true
2020/04/05 12:45:25 [debug] nginx_proxy_directives = {}
2020/04/05 12:45:25 [debug] nginx_sproxy_directives = {}
2020/04/05 12:45:25 [debug] nginx_stream_directives = {}
2020/04/05 12:45:25 [debug] nginx_user = "nobody nobody"
2020/04/05 12:45:25 [debug] nginx_worker_processes = "auto"
2020/04/05 12:45:25 [debug] origins = {}
2020/04/05 12:45:25 [debug] pg_database = "kong"
2020/04/05 12:45:25 [debug] pg_host = "127.0.0.1"
2020/04/05 12:45:25 [debug] pg_port = 5432
2020/04/05 12:45:25 [debug] pg_ssl = false
2020/04/05 12:45:25 [debug] pg_ssl_verify = false
2020/04/05 12:45:25 [debug] pg_timeout = 5000
2020/04/05 12:45:25 [debug] pg_user = "kong"
2020/04/05 12:45:25 [debug] plugins = {"bundled"}
2020/04/05 12:45:25 [debug] prefix = "/usr/local/kong/"
2020/04/05 12:45:25 [debug] proxy_access_log = "logs/access.log"
2020/04/05 12:45:25 [debug] proxy_error_log = "logs/error.log"
2020/04/05 12:45:25 [debug] proxy_listen = {"0.0.0.0:8000","0.0.0.0:8443 ssl"}
2020/04/05 12:45:25 [debug] real_ip_header = "X-Real-IP"
2020/04/05 12:45:25 [debug] real_ip_recursive = "off"
2020/04/05 12:45:25 [debug] ssl_cipher_suite = "modern"
2020/04/05 12:45:25 [debug] ssl_ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"
2020/04/05 12:45:25 [debug] stream_listen = {"off"}
2020/04/05 12:45:25 [debug] trusted_ips = {}
2020/04/05 12:45:25 [debug] upstream_keepalive = 60
2020/04/05 12:45:25 [debug] validate_certs_refresh_interval = 300
2020/04/05 12:45:25 [debug] validate_certs_warn_before = 7
2020/04/05 12:45:25 [verbose] prefix in use: /usr/local/kong
2020/04/05 12:45:25 [error] could not resolve Cassandra contact point 'ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local': failed to create a resolver: failed to set peer name: host or service not provided, or not known
Error:
/usr/local/share/lua/5.1/kong/cmd/start.lua:42: could not resolve any of the provided Cassandra contact points (cassandra_contact_points = 'ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local')
stack traceback:
        [C]: in function 'assert'
        /usr/local/share/lua/5.1/kong/cmd/start.lua:42: in function 'cmd_exec'
        /usr/local/share/lua/5.1/kong/cmd/init.lua:90: in function </usr/local/share/lua/5.1/kong/cmd/init.lua:90>
        [C]: in function 'xpcall'
        /usr/local/share/lua/5.1/kong/cmd/init.lua:90: in function </usr/local/share/lua/5.1/kong/cmd/init.lua:47>
        /etc/kong/openresty_temp_script.lua:6: in function 'file_gen'
        init_worker_by_lua:48: in function <init_worker_by_lua:46>
        [C]: in function 'xpcall'
        init_worker_by_lua:55: in function <init_worker_by_lua:53>

Please note that getent can find the proper IP for cassandra service name:

bash-4.2$ getent hosts ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local
2001:283:4000:2001::5167 ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local

bash-4.2$ cat /etc/resolv.conf
nameserver fc00::13
search csfapiexp-ccas.svc.cluster.local csfapiexp-ckng.svc.cluster.local svc.cluster.local cluster.local bcmt.cluster.local
options ndots:5

I cannot find any Kong documentation that desribes how to configure configure Kong 1.1.x to support IPv6 connections to Cassandra db. Please help.

I appreciate any suggestions,
Ania

[angrosz]

another case when KONG_CASSANDRA_CONTACT_POINTS is one IPv6 address

Error:
/usr/local/share/lua/5.1/kong/cmd/migrations.lua:107: [Cassandra error] all hosts tried for query failed. [2001:283:4000:2001::5158]: host seems unhealthy, considering it down (host or service not provided, or not known)

[root@bcmt-sandbox4-3c3w2e-2003-control-01 angrosz]# kubectl logs -f -n csfapiexp-ckng ckng-ckng-init-migrations-q566t
2020/04/01 10:39:16 Provided contact points: ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local
2020/04/01 10:39:16 CONTACT POINT: [ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local]
2020/04/01 10:39:16 CONTACT IPs: [2001:283:4000:2001::5158]
2020/04/01 10:39:16 Resolved KONG_CASSANDRA_CONTACT_POINTS: 2001:283:4000:2001::5158
2020/04/01 10:39:16 Executing kong migrations bootstrap. It could take some time, so please be patient...
2020/04/01 10:39:16 [verbose] Kong: 1.1.1
2020/04/01 10:39:16 [debug] ngx_lua: 10013
2020/04/01 10:39:16 [debug] nginx: 1013006
2020/04/01 10:39:16 [debug] Lua: LuaJIT 2.1.0-beta3
2020/04/01 10:39:16 [verbose] no config file found at /etc/kong/kong.conf
2020/04/01 10:39:16 [verbose] no config file, skip loading
2020/04/01 10:39:16 [debug] reading environment variables
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_USERNAME ENV found with "suadmincass"
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_PASSWORD ENV found with "******"
2020/04/01 10:39:16 [debug] KONG_DB_UPDATE_PROPAGATION ENV found with "1"
2020/04/01 10:39:16 [debug] KONG_DATABASE ENV found with "cassandra"
2020/04/01 10:39:16 [debug] KONG_LARGE_CLIENT_HEADER_BUFFERS_NUMBER ENV found with "4"
2020/04/01 10:39:16 [debug] KONG_CLIENT_HEADER_BUFFER_SIZE ENV found with "1k"
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_SCHEMA_CONSENSUS_TIMEOUT ENV found with "180000"
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_CONTACT_POINTS ENV found with "2001:283:4000:2001::5158"
2020/04/01 10:39:16 [debug] KONG_LARGE_CLIENT_HEADER_BUFFERS_SIZE ENV found with "8k"
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_REPL_FACTOR ENV found with "1"
2020/04/01 10:39:16 [debug] KONG_NGINX_DAEMON ENV found with "off"
2020/04/01 10:39:16 [debug] KONG_ANONYMOUS_REPORTS ENV found with "off"
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_CONSISTENCY ENV found with "QUORUM"
2020/04/01 10:39:16 [debug] admin_access_log = "logs/admin_access.log"
2020/04/01 10:39:16 [debug] admin_api_spec_listen = {"0.0.0.0:8080","0.0.0.0:9443 ssl"}
2020/04/01 10:39:16 [debug] admin_error_log = "logs/error.log"
2020/04/01 10:39:16 [debug] admin_listen = {"0.0.0.0:8001","0.0.0.0:8444 ssl"}
2020/04/01 10:39:16 [debug] anonymous_reports = false
2020/04/01 10:39:16 [debug] cache_negative_ttl_default = 60
2020/04/01 10:39:16 [debug] cache_negative_ttl_max = 300
2020/04/01 10:39:16 [debug] cache_negative_ttl_min = 1
2020/04/01 10:39:16 [debug] cache_ttl_default = 3600
2020/04/01 10:39:16 [debug] cache_ttl_max = 3600
2020/04/01 10:39:16 [debug] cache_ttl_min = 60
2020/04/01 10:39:16 [debug] cassandra_cluster_refresh_interval = 15
2020/04/01 10:39:16 [debug] cassandra_consistency = "QUORUM"
2020/04/01 10:39:16 [debug] cassandra_contact_points = {"2001:283:4000:2001::5158"}
2020/04/01 10:39:16 [debug] cassandra_data_centers = {"dc1:2","dc2:3"}
2020/04/01 10:39:16 [debug] cassandra_keyspace = "kong"
2020/04/01 10:39:16 [debug] cassandra_lb_policy = "RequestRoundRobin"
2020/04/01 10:39:16 [debug] cassandra_password = "******"
2020/04/01 10:39:16 [debug] cassandra_port = 9042
2020/04/01 10:39:16 [debug] cassandra_repl_factor = 1
2020/04/01 10:39:16 [debug] cassandra_repl_strategy = "SimpleStrategy"
2020/04/01 10:39:16 [debug] cassandra_schema_consensus_timeout = 180000
2020/04/01 10:39:16 [debug] cassandra_ssl = false
2020/04/01 10:39:16 [debug] cassandra_ssl_verify = false
2020/04/01 10:39:16 [debug] cassandra_timeout = 5000
2020/04/01 10:39:16 [debug] cassandra_username = "suadmincass"
2020/04/01 10:39:16 [debug] client_body_buffer_size = "8k"
2020/04/01 10:39:16 [debug] client_header_buffer_size = "1k"
2020/04/01 10:39:16 [debug] client_max_body_size = "0"
2020/04/01 10:39:16 [debug] client_ssl = false
2020/04/01 10:39:16 [debug] database = "cassandra"
2020/04/01 10:39:16 [debug] db_cache_ttl = 0
2020/04/01 10:39:16 [debug] db_resurrect_ttl = 30
2020/04/01 10:39:16 [debug] db_update_frequency = 5
2020/04/01 10:39:16 [debug] db_update_propagation = 1
2020/04/01 10:39:16 [debug] dns_error_ttl = 1
2020/04/01 10:39:16 [debug] dns_hostsfile = "/etc/hosts"
2020/04/01 10:39:16 [debug] dns_no_sync = false
2020/04/01 10:39:16 [debug] dns_not_found_ttl = 30
2020/04/01 10:39:16 [debug] dns_order = {"LAST","SRV","A","CNAME"}
2020/04/01 10:39:16 [debug] dns_resolver = {}
2020/04/01 10:39:16 [debug] dns_stale_ttl = 4
2020/04/01 10:39:16 [debug] error_default_type = "text/plain"
2020/04/01 10:39:16 [debug] headers = {"server_tokens","latency_tokens"}
2020/04/01 10:39:16 [debug] large_client_header_buffers_number = 4
Error:
/usr/local/share/lua/5.1/kong/cmd/migrations.lua:107: [Cassandra error] all hosts tried for query failed. [2001:283:4000:2001::5158]: host seems unhealthy, considering it down (host or service not provided, or not known)
stack traceback:
        [C]: in function 'assert'
        /usr/local/share/lua/5.1/kong/cmd/migrations.lua:107: in function 'cmd_exec'
        /usr/local/share/lua/5.1/kong/cmd/init.lua:90: in function </usr/local/share/lua/5.1/kong/cmd/init.lua:90>
        [C]: in function 'xpcall'
        /usr/local/share/lua/5.1/kong/cmd/init.lua:90: in function </usr/local/share/lua/5.1/kong/cmd/init.lua:47>
        /etc/kong/openresty_temp_script.lua:6: in function 'file_gen'
        init_worker_by_lua:49: in function <init_worker_by_lua:47>
        [C]: in function 'xpcall'
        init_worker_by_lua:56: in function <init_worker_by_lua:54>
2020/04/01 10:39:16 [debug] large_client_header_buffers_size = "8k"
2020/04/01 10:39:16 [debug] log_anonymize = "OFF"
2020/04/01 10:39:16 [debug] log_level = "notice"
2020/04/01 10:39:16 [debug] lru_cache_size = 500000
2020/04/01 10:39:16 [debug] lua_package_cpath = ""
2020/04/01 10:39:16 [debug] lua_package_path = "./?.lua;./?/init.lua;"
2020/04/01 10:39:16 [debug] lua_socket_pool_size = 30
2020/04/01 10:39:16 [debug] lua_ssl_verify_depth = 1
2020/04/01 10:39:16 [debug] maria_database = "kong"
2020/04/01 10:39:16 [debug] maria_host = "127.0.0.1"
2020/04/01 10:39:16 [debug] maria_password = "******"
2020/04/01 10:39:16 [debug] maria_port = 3306
2020/04/01 10:39:16 [debug] maria_socket_keepalive_timeout = 60000
2020/04/01 10:39:16 [debug] maria_socket_pool_size = 20
2020/04/01 10:39:16 [debug] maria_ssl = false
2020/04/01 10:39:16 [debug] maria_ssl_verify = false
2020/04/01 10:39:16 [debug] maria_timeout = 10000
2020/04/01 10:39:16 [debug] maria_user = "kong"
2020/04/01 10:39:16 [debug] mem_cache_size = "128m"
2020/04/01 10:39:16 [debug] nginx_admin_directives = {}
2020/04/01 10:39:16 [debug] nginx_daemon = "off"
2020/04/01 10:39:16 [debug] nginx_http_directives = {}
2020/04/01 10:39:16 [debug] nginx_optimizations = true
2020/04/01 10:39:16 [debug] nginx_proxy_directives = {}
2020/04/01 10:39:16 [debug] nginx_sproxy_directives = {}
2020/04/01 10:39:16 [debug] nginx_stream_directives = {}
2020/04/01 10:39:16 [debug] nginx_user = "nobody nobody"
2020/04/01 10:39:16 [debug] nginx_worker_processes = "auto"
2020/04/01 10:39:16 [debug] origins = {}
2020/04/01 10:39:16 [debug] pg_database = "kong"
2020/04/01 10:39:16 [debug] pg_host = "127.0.0.1"
2020/04/01 10:39:16 [debug] pg_port = 5432
2020/04/01 10:39:16 [debug] pg_ssl = false
2020/04/01 10:39:16 [debug] pg_ssl_verify = false
2020/04/01 10:39:16 [debug] pg_timeout = 5000
2020/04/01 10:39:16 [debug] pg_user = "kong"
2020/04/01 10:39:16 [debug] plugins = {"bundled"}
2020/04/01 10:39:16 [debug] prefix = "/usr/local/kong/"
2020/04/01 10:39:16 [debug] proxy_access_log = "logs/access.log"
2020/04/01 10:39:16 [debug] proxy_error_log = "logs/error.log"
2020/04/01 10:39:16 [debug] proxy_listen = {"0.0.0.0:8000","0.0.0.0:8443 ssl"}
2020/04/01 10:39:16 [debug] real_ip_header = "X-Real-IP"
2020/04/01 10:39:16 [debug] real_ip_recursive = "off"
2020/04/01 10:39:16 [debug] ssl_cipher_suite = "modern"
2020/04/01 10:39:16 [debug] ssl_ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"
2020/04/01 10:39:16 [debug] stream_listen = {"off"}
2020/04/01 10:39:16 [debug] trusted_ips = {}
2020/04/01 10:39:16 [debug] upstream_keepalive = 60
2020/04/01 10:39:16 [debug] validate_certs_refresh_interval = 300
2020/04/01 10:39:16 [debug] validate_certs_warn_before = 7
2020/04/01 10:39:16 [verbose] prefix in use: /usr/local/kong
2020/04/01 10:39:16 [debug] resolved Cassandra contact point '2001:283:4000:2001::5158' to: [2001:283:4000:2001::5158]
2020/04/01 10:39:16 Executing kong pre-upgrade step.
Error: [Cassandra error] all hosts tried for query failed. [2001:283:4000:2001::5158]: host seems unhealthy, considering it down (host or service not provided, or not known)  Run with --v (verbose) or --vv (debug) for more details
2020/04/01 10:39:16 Kong pre-upgrade step [kong su_check] failed.

[angrosz]

another case when KONG_CASSANDRA_CONTACT_POINTS is set to 3 IPv6 addresses

Error:
/usr/local/share/lua/5.1/kong/cmd/migrations.lua:34: [cassandra error] could not find coordinator: all hosts tried for query failed. 2001:283:4000:2001::5151: host seems unhealthy, considering it down (Address family for hostname not supported). 2001:283:4000:2001::3230: host seems unhealthy, considering it down (Address family for hostname not supported). 2001:283:4000:2001::3077: host seems unhealthy, considering it down (Address family for hostname not supported)

bash-4.2$ kong migrations up --vv
2020/03/31 20:31:40 [verbose] Kong: 0.13.1
2020/03/31 20:31:40 [debug] ngx_lua: 10010
2020/03/31 20:31:40 [debug] nginx: 1011002
2020/03/31 20:31:40 [debug] Lua: LuaJIT 2.1.0-beta3
2020/03/31 20:31:40 [verbose] no config file found at /etc/kong/kong.conf
2020/03/31 20:31:40 [verbose] no config file, skipping loading
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_USERNAME ENV found with "suadmincass"
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_PASSWORD ENV found with "******"
2020/03/31 20:31:40 [debug] KONG_DB_UPDATE_PROPAGATION ENV found with "1"
2020/03/31 20:31:40 [debug] KONG_DATABASE ENV found with "cassandra"
2020/03/31 20:31:40 [debug] KONG_LARGE_CLIENT_HEADER_BUFFERS_NUMBER ENV found with "4"
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_SCHEMA_CONSENSUS_TIMEOUT ENV found with "180000"
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_CONTACT_POINTS ENV found with "2001:283:4000:2001::5151,2001:283:4000:2001::3230,2001:283:4000:2001::3077"
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_REPL_FACTOR ENV found with "3"
2020/03/31 20:31:40 [debug] KONG_CLIENT_HEADER_BUFFER_SIZE ENV found with "1k"
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_CONSISTENCY ENV found with "QUORUM"
2020/03/31 20:31:40 [debug] KONG_NGINX_DAEMON ENV found with "off"
2020/03/31 20:31:40 [debug] KONG_LARGE_CLIENT_HEADER_BUFFERS_SIZE ENV found with "8k"
2020/03/31 20:31:40 [debug] KONG_ANONYMOUS_REPORTS ENV found with "off"
2020/03/31 20:31:40 [debug] admin_access_log = "logs/admin_access.log"
2020/03/31 20:31:40 [debug] admin_api_spec_listen = {"0.0.0.0:8080","0.0.0.0:9443 ssl"}
2020/03/31 20:31:40 [debug] admin_error_log = "logs/error.log"
2020/03/31 20:31:40 [debug] admin_listen = {"0.0.0.0:8001","0.0.0.0:8444 ssl"}
2020/03/31 20:31:40 [debug] anonymous_reports = false
2020/03/31 20:31:40 [debug] cache_negative_ttl_default = 60
2020/03/31 20:31:40 [debug] cache_negative_ttl_max = 300
2020/03/31 20:31:40 [debug] cache_negative_ttl_min = 1
2020/03/31 20:31:40 [debug] cache_ttl_default = 3600
2020/03/31 20:31:40 [debug] cache_ttl_max = 3600
2020/03/31 20:31:40 [debug] cache_ttl_min = 60
2020/03/31 20:31:40 [debug] cassandra_cluster_refresh_interval = 15
2020/03/31 20:31:40 [debug] cassandra_consistency = "QUORUM"
2020/03/31 20:31:40 [debug] cassandra_contact_points = {"2001:283:4000:2001::5151","2001:283:4000:2001::3230","2001:283:4000:2001::3077"}
2020/03/31 20:31:40 [debug] cassandra_data_centers = {"dc1:2","dc2:3"}
2020/03/31 20:31:40 [debug] cassandra_keyspace = "kong"
2020/03/31 20:31:40 [debug] cassandra_lb_policy = "RoundRobin"
2020/03/31 20:31:40 [debug] cassandra_password = "******"
2020/03/31 20:31:40 [debug] cassandra_port = 9042
2020/03/31 20:31:40 [debug] cassandra_repl_factor = 3
2020/03/31 20:31:40 [debug] cassandra_repl_strategy = "SimpleStrategy"
2020/03/31 20:31:40 [debug] cassandra_schema_consensus_timeout = 180000
2020/03/31 20:31:40 [debug] cassandra_ssl = false
2020/03/31 20:31:40 [debug] cassandra_ssl_verify = false
2020/03/31 20:31:40 [debug] cassandra_timeout = 5000
2020/03/31 20:31:40 [debug] cassandra_username = "suadmincass"
2020/03/31 20:31:40 [debug] client_body_buffer_size = "8k"
2020/03/31 20:31:40 [debug] client_header_buffer_size = "1k"
2020/03/31 20:31:40 [debug] client_max_body_size = "0"
2020/03/31 20:31:40 [debug] client_ssl = false
2020/03/31 20:31:40 [debug] custom_plugins = {}
2020/03/31 20:31:40 [debug] database = "cassandra"
2020/03/31 20:31:40 [debug] db_cache_ttl = 3600
2020/03/31 20:31:40 [debug] db_update_frequency = 5
2020/03/31 20:31:40 [debug] db_update_propagation = 1
2020/03/31 20:31:40 [debug] dns_error_ttl = 1
2020/03/31 20:31:40 [debug] dns_hostsfile = "/etc/hosts"
2020/03/31 20:31:40 [debug] dns_no_sync = false
2020/03/31 20:31:40 [debug] dns_not_found_ttl = 30
2020/03/31 20:31:40 [debug] dns_order = {"LAST","SRV","A","CNAME"}
2020/03/31 20:31:40 [debug] dns_resolver = {}
2020/03/31 20:31:40 [debug] dns_stale_ttl = 4
2020/03/31 20:31:40 [debug] error_default_type = "text/plain"
2020/03/31 20:31:40 [debug] large_client_header_buffers_number = 4
Error:
/usr/local/share/lua/5.1/kong/cmd/migrations.lua:34: [cassandra error] could not find coordinator: all hosts tried for query failed. 2001:283:4000:2001::5151: host seems unhealthy, considering it down (Address family for hostname not supported). 2001:283:4000:2001::3230: host seems unhealthy, considering it down (Address family for hostname not supported). 2001:283:4000:2001::3077: host seems unhealthy, considering it down (Address family for hostname not supported)
stack traceback:
        [C]: in function 'assert'
        /usr/local/share/lua/5.1/kong/cmd/migrations.lua:34: in function 'cmd_exec'
        /usr/local/share/lua/5.1/kong/cmd/init.lua:87: in function </usr/local/share/lua/5.1/kong/cmd/init.lua:87>
        [C]: in function 'xpcall'
        /usr/local/share/lua/5.1/kong/cmd/init.lua:87: in function </usr/local/share/lua/5.1/kong/cmd/init.lua:44>
        /usr/local/bin/kong:7: in function 'file_gen'
        init_worker_by_lua:46: in function <init_worker_by_lua:44>
        [C]: in function 'xpcall'
        init_worker_by_lua:53: in function <init_worker_by_lua:51>
2020/03/31 20:31:40 [debug] large_client_header_buffers_size = "8k"
2020/03/31 20:31:40 [debug] latency_tokens = true
2020/03/31 20:31:40 [debug] log_level = "notice"
2020/03/31 20:31:40 [debug] lru_cache_size = 500000
2020/03/31 20:31:40 [debug] lua_package_cpath = ""
2020/03/31 20:31:40 [debug] lua_package_path = "./?.lua;./?/init.lua;"
2020/03/31 20:31:40 [debug] lua_socket_pool_size = 30
2020/03/31 20:31:40 [debug] lua_ssl_verify_depth = 1
2020/03/31 20:31:40 [debug] maria_database = "kong"
2020/03/31 20:31:40 [debug] maria_host = "127.0.0.1"
2020/03/31 20:31:40 [debug] maria_password = "******"
2020/03/31 20:31:40 [debug] maria_port = 3306
2020/03/31 20:31:40 [debug] maria_socket_keepalive_timeout = 60000
2020/03/31 20:31:40 [debug] maria_socket_pool_size = 20
2020/03/31 20:31:40 [debug] maria_ssl = false
2020/03/31 20:31:40 [debug] maria_ssl_verify = false
2020/03/31 20:31:40 [debug] maria_timeout = 3000
2020/03/31 20:31:40 [debug] maria_user = "kong"
2020/03/31 20:31:40 [debug] mem_cache_size = "128m"
2020/03/31 20:31:40 [debug] nginx_daemon = "off"
2020/03/31 20:31:40 [debug] nginx_optimizations = true
2020/03/31 20:31:40 [debug] nginx_user = "nobody nobody"
2020/03/31 20:31:40 [debug] nginx_worker_processes = "auto"
2020/03/31 20:31:40 [debug] pg_database = "kong"
2020/03/31 20:31:40 [debug] pg_host = "127.0.0.1"
2020/03/31 20:31:40 [debug] pg_port = 5432
2020/03/31 20:31:40 [debug] pg_ssl = false
2020/03/31 20:31:40 [debug] pg_ssl_verify = false
2020/03/31 20:31:40 [debug] pg_user = "kong"
2020/03/31 20:31:40 [debug] prefix = "/usr/local/kong/"
2020/03/31 20:31:40 [debug] proxy_access_log = "logs/access.log"
2020/03/31 20:31:40 [debug] proxy_error_log = "logs/error.log"
2020/03/31 20:31:40 [debug] proxy_listen = {"0.0.0.0:8000","0.0.0.0:8443 ssl"}
2020/03/31 20:31:40 [debug] real_ip_header = "X-Real-IP"
2020/03/31 20:31:40 [debug] real_ip_recursive = "off"
2020/03/31 20:31:40 [debug] server_tokens = true
2020/03/31 20:31:40 [debug] ssl_cipher_suite = "modern"
2020/03/31 20:31:40 [debug] ssl_ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"
2020/03/31 20:31:40 [debug] trusted_ips = {}
2020/03/31 20:31:40 [debug] upstream_keepalive = 60
2020/03/31 20:31:40 [verbose] prefix in use: /usr/local/kong
2020/03/31 20:31:40 [verbose] running datastore migrations
2020/03/31 20:31:40 Kong migrations failed.