Fixes 14

README.md

@@ -43,7 +43,7 @@ only makes sense if you trust your DoH provider.
 
 ## Build
 
-Depends on `c-ares (>=1.11.0)`, `libcurl (>=7.64.0)`, `libev (>=4.25)`.
+Depends on `c-ares (>=1.11.0)`, `libcurl (>=7.65.0)`, `libev (>=4.25)`.
 
 On Debian-derived systems those are libc-ares-dev,
 libcurl4-{openssl,nss,gnutls}-dev and libev-dev respectively.
@@ -159,40 +159,55 @@ Just run it as a daemon and point traffic at it. Commandline flags are:
 
 ```
 Usage: ./https_dns_proxy [-a <listen_addr>] [-p <listen_port>]
-        [-d] [-u <user>] [-g <group>] [-b <dns_servers>]
-        [-i <polling_interval>] [-4] [-r <resolver_url>]
-        [-t <proxy_server>] [-l <logfile>] [-c <dscp_codepoint>]
-        [-x] [-q] [-s <statistic_interval>] [-v]+ [-V] [-h]
+        [-b <dns_servers>] [-i <polling_interval>] [-4]
+        [-r <resolver_url>] [-t <proxy_server>] [-x] [-q] [-C <ca_path>] [-c <dscp_codepoint>]
+        [-d] [-u <user>] [-g <group>]
+        [-v]+ [-l <logfile>] [-s <statistic_interval>] [-F <log_limit>] [-V] [-h]
 
-  -a listen_addr         Local IPv4/v6 address to bind to. (127.0.0.1)
-  -p listen_port         Local port to bind to. (5053)
-  -d                     Daemonize.
-  -u user                Optional user to drop to if launched as root.
-  -g group               Optional group to drop to if launched as root.
+ DNS server
+  -a listen_addr         Local IPv4/v6 address to bind to. (Default: 127.0.0.1)
+  -p listen_port         Local port to bind to. (Default: 5053)
+
+ DNS client
   -b dns_servers         Comma-separated IPv4/v6 addresses and ports (addr:port)
                          of DNS servers to resolve resolver host (e.g. dns.google).
                          When specifying a port for IPv6, enclose the address in [].
-                         (8.8.8.8,1.1.1.1,8.8.4.4,1.0.0.1,145.100.185.15,145.100.185.16,185.49.141.37)
+                         (Default: 8.8.8.8,1.1.1.1,8.8.4.4,1.0.0.1,145.100.185.15,145.100.185.16,185.49.141.37)
   -i polling_interval    Optional polling interval of DNS servers.
                          (Default: 120, Min: 5, Max: 3600)
   -4                     Force IPv4 hostnames for DNS resolvers non IPv6 networks.
-  -r resolver_url        The HTTPS path to the resolver URL. Default: https://dns.google/dns-query
+
+ HTTPS client
+  -r resolver_url        The HTTPS path to the resolver URL. (Default: https://dns.google/dns-query)
   -t proxy_server        Optional HTTP proxy. e.g. socks5://127.0.0.1:1080
                          Remote name resolution will be used if the protocol
                          supports it (http, https, socks4a, socks5h), otherwise
                          initial DNS resolution will still be done via the
                          bootstrap DNS servers.
-  -l logfile             Path to file to log to. ("-")
-  -c dscp_codepoint      Optional DSCP codepoint[0-63] to set on upstream DNS server
-                         connections.
   -x                     Use HTTP/1.1 instead of HTTP/2. Useful with broken
-                         or limited builds of libcurl. (false)
-  -q                     Use HTTP/3 (QUIC) only. (false)
-  -s statistic_interval  Optional statistic printout interval.
-                         (Default: 0, Disabled: 0, Min: 1, Max: 3600)
+                         or limited builds of libcurl.
+  -q                     Use HTTP/3 (QUIC) only.
+  -m max_idle_time       Maximum idle time in seconds allowed for reusing a HTTPS connection.
+                         (Default: 118, Min: 0, Max: 3600)
+  -C ca_path             Optional file containing CA certificates.
+  -c dscp_codepoint      Optional DSCP codepoint to set on upstream HTTPS server
+                         connections. (Min: 0, Max: 63)
+
+ Process
+  -d                     Daemonize.
+  -u user                Optional user to drop to if launched as root.
+  -g group               Optional group to drop to if launched as root.
+
+ Logging
   -v                     Increase logging verbosity. (Default: error)
                          Levels: fatal, stats, error, warning, info, debug
                          Request issues are logged on warning level.
+  -l logfile             Path to file to log to. (Default: standard output)
+  -s statistic_interval  Optional statistic printout interval.
+                         (Default: 0, Disabled: 0, Min: 1, Max: 3600)
+  -F log_limit           Flight recorder: storing desired amount of logs from all levels
+                         in memory and dumping them on fatal error or on SIGUSR2 signal.
+                         (Default: 0, Disabled: 0, Min: 100, Max: 100000)
   -V                     Print version and exit.
   -h                     Print help and exit.
 ```

development_build_with_http3.sh

@@ -7,49 +7,54 @@ echo "WARNING !!!"
 echo
 echo "Use only for development and testing!"
 echo "It is highly highly not recommended, to use in production!"
-echo "This script was based on: https://github.com/curl/curl/blob/curl-7_82_0/docs/HTTP3.md"
+echo "This script was based on: https://github.com/curl/curl/blob/curl-8_12_1/docs/HTTP3.md"
+echo
+echo "Extra packages suggested to be installed: autoconf libtool"
 echo
 
 sleep 5
 
+set -x
+
 INSTALL_DIR=$PWD/custom_curl/install
 mkdir -p $INSTALL_DIR
 cd custom_curl
 
 ###
 
-git clone --depth 1 -b openssl-3.0.0+quic https://github.com/quictls/openssl
+git clone --depth 1 -b openssl-3.1.4+quic https://github.com/quictls/openssl
 cd openssl
 ./config enable-tls1_3 --prefix=$INSTALL_DIR
 make -j build_libs
 make install_dev
 cd ..
 
-git clone --depth 1 -b v0.3.0 https://github.com/ngtcp2/nghttp3
+git clone --depth 1 -b v1.1.0 https://github.com/ngtcp2/nghttp3
 cd nghttp3
+git submodule update --init
 autoreconf -fi
 ./configure --prefix=$INSTALL_DIR --enable-lib-only
 make -j
 make install
 cd ..
 
-git clone --depth 1 -b v0.3.1 https://github.com/ngtcp2/ngtcp2
+git clone --depth 1 -b v1.2.0 https://github.com/ngtcp2/ngtcp2
 cd ngtcp2
 autoreconf -fi
 ./configure PKG_CONFIG_PATH=$INSTALL_DIR/lib64/pkgconfig:$INSTALL_DIR/lib64/pkgconfig LDFLAGS="-Wl,-rpath,$INSTALL_DIR/lib64" --prefix=$INSTALL_DIR --enable-lib-only --with-openssl
 make -j
 make install
 cd ..
 
-git clone --depth 1 -b v1.47.0 https://github.com/nghttp2/nghttp2
+git clone --depth 1 -b v1.64.0 https://github.com/nghttp2/nghttp2
 cd nghttp2
 autoreconf -fi
 ./configure PKG_CONFIG_PATH=$INSTALL_DIR/lib64/pkgconfig:$INSTALL_DIR/lib64/pkgconfig LDFLAGS="-Wl,-rpath,$INSTALL_DIR/lib64" --prefix=$INSTALL_DIR --enable-lib-only --with-openssl
 make -j
 make install
 cd ..
 
-git clone --depth 1 -b curl-7_82_0 https://github.com/curl/curl
+git clone --depth 1 -b curl-8_12_1 https://github.com/curl/curl
 cd curl
 autoreconf -fi
 LDFLAGS="-Wl,-rpath,$INSTALL_DIR/lib64" ./configure --with-openssl=$INSTALL_DIR --with-nghttp2=$INSTALL_DIR --with-nghttp3=$INSTALL_DIR --with-ngtcp2=$INSTALL_DIR --prefix=$INSTALL_DIR
@@ -60,5 +65,5 @@ cd ..
 ###
 
 cd ..
-cmake -D CUSTOM_LIBCURL_INSTALL_PATH=$INSTALL_DIR .
+cmake -D CUSTOM_LIBCURL_INSTALL_PATH=$INSTALL_DIR -D CMAKE_BUILD_TYPE=Debug .
 make -j

src/dns_poller.c

@@ -35,7 +35,7 @@ static void sock_state_cb(void *data, int fd, int read, int write) {
   // reserve and start new event on unused slot
   io_event_ptr = get_io_event(d, 0);
   if (!io_event_ptr) {
-    FLOG("c-ares needed more event, than nameservers count: %d", d->io_events_count);
+    FLOG("c-ares needed more IO event handler, than the number of provided nameservers: %d", d->io_events_count);
   }
   DLOG("Reserved new io event: %p", io_event_ptr);
   // NOLINTNEXTLINE(clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling)
@@ -75,9 +75,9 @@ static void ares_cb(void *arg, int status, int __attribute__((unused)) timeouts,
   ev_tstamp interval = 5;  // retry by default after some time
 
   if (status != ARES_SUCCESS) {
-    WLOG("DNS lookup failed: %s", ares_strerror(status));
+    WLOG("DNS lookup of '%s' failed: %s", d->hostname, ares_strerror(status));
   } else if (!h || h->h_length < 1) {
-    WLOG("No hosts.");
+    WLOG("No hosts found for '%s'", d->hostname);
   } else {
     interval = d->polling_interval;
     d->cb(d->hostname, d->cb_data, get_addr_listing(h->h_addr_list, h->h_addrtype));

src/https_client.c

@@ -50,6 +50,11 @@ DOH_MAX_RESPONSE_SIZE = 65535
     FLOG("Unexpected NULL pointer for " #var_name "(" #type ")"); \
   }
 
+static void https_fetch_ctx_cleanup(https_client_t *client,
+                                    struct https_fetch_ctx *prev,
+                                    struct https_fetch_ctx *ctx,
+                                    int curl_result_code);
+
 static size_t write_buffer(void *buf, size_t size, size_t nmemb, void *userp) {
   GET_PTR(struct https_fetch_ctx, ctx, userp);
   size_t write_size = size * nmemb;
@@ -76,14 +81,20 @@ static curl_socket_t opensocket_callback(void *clientp, curlsocktype purpose,
                                          struct curl_sockaddr *addr) {
   GET_PTR(https_client_t, client, clientp);
 
+  if (client->connections >= HTTPS_SOCKET_LIMIT) {
+    ELOG("curl needed more socket, than the number of maximum sockets: %d", HTTPS_SOCKET_LIMIT);
+    return CURL_SOCKET_BAD;
+  }
+
   curl_socket_t sock = socket(addr->family, addr->socktype, addr->protocol);
-  if (sock != -1) {
-    DLOG("curl opened socket: %d", sock);
-  } else {
+  if (sock == -1) {
     ELOG("Could not open curl socket %d:%s", errno, strerror(errno));
     return CURL_SOCKET_BAD;
   }
 
+  DLOG("curl opened socket: %d", sock);
+  client->connections++;
+
   if (client->stat) {
     stat_connection_opened(client->stat);
   }
@@ -112,13 +123,14 @@ static int closesocket_callback(void __attribute__((unused)) *clientp, curl_sock
 {
   GET_PTR(https_client_t, client, clientp);
 
-  if (close(sock) == 0) {
-    DLOG("curl closed socket: %d", sock);
-  } else {
+  if (close(sock) != 0) {
     ELOG("Could not close curl socket %d:%s", errno, strerror(errno));
     return 1;
   }
 
+  DLOG("curl closed socket: %d", sock);
+  client->connections--;
+
   if (client->stat) {
     stat_connection_closed(client->stat);
   }
@@ -265,7 +277,6 @@ static void https_set_request_version(https_client_t *client,
     } else if (client->opt->use_http_version == 2) {
       ELOG("Try to run application with -x argument! Falling back to HTTP/1.1 version.");
       client->opt->use_http_version = 1;
-      // TODO: consider CURLMOPT_PIPELINING setting??
     }
   }
 }
@@ -293,30 +304,22 @@ static void https_fetch_ctx_init(https_client_t *client,
     ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_DEBUGFUNCTION, https_curl_debug);
     ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_DEBUGDATA, ctx);
   }
-  if (logging_debug_enabled() || client->stat || client->opt->dscp) {
-    ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_OPENSOCKETFUNCTION, opensocket_callback);
-    ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_OPENSOCKETDATA, client);
-  }
-  if (logging_debug_enabled() || client->stat) {
-    ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_CLOSESOCKETFUNCTION, closesocket_callback);
-    ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_CLOSESOCKETDATA, client);
-  }
+  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_OPENSOCKETFUNCTION, opensocket_callback);
+  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_OPENSOCKETDATA, client);
+  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_CLOSESOCKETFUNCTION, closesocket_callback);
+  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_CLOSESOCKETDATA, client);
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_URL, url);
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_HTTPHEADER, client->header_list);
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_POSTFIELDSIZE, datalen);
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_POSTFIELDS, data);
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_WRITEFUNCTION, &write_buffer);
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_WRITEDATA, ctx);
-#ifdef CURLOPT_MAXAGE_CONN
-  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_TCP_KEEPALIVE, 1L);
-  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_TCP_KEEPIDLE, 50L);
-  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_TCP_KEEPINTVL, 50L);
-  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_MAXAGE_CONN, 300L);
-#endif
+  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_MAXAGE_CONN, client->opt->max_idle_time);
+  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_PIPEWAIT, client->opt->use_http_version > 1);
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_USERAGENT, "https_dns_proxy/0.3");
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_FOLLOWLOCATION, 0);
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_NOSIGNAL, 0);
-  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_TIMEOUT, 10 /* seconds */);
+  ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_TIMEOUT, client->connections > 0 ? 5 : 10 /* seconds */);
   // We know Google supports this, so force it.
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_ERRORBUFFER, ctx->curl_errbuf); // zeroed by calloc
@@ -329,8 +332,13 @@ static void https_fetch_ctx_init(https_client_t *client,
   }
   CURLMcode multi_code = curl_multi_add_handle(client->curlm, ctx->curl);
   if (multi_code != CURLM_OK) {
-    FLOG_REQ("curl_multi_add_handle error %d: %s",
-             multi_code, curl_multi_strerror(multi_code));
+    ELOG_REQ("curl_multi_add_handle error %d: %s", multi_code, curl_multi_strerror(multi_code));
+    if (multi_code == CURLM_ABORTED_BY_CALLBACK) {
+      WLOG_REQ("Resetting HTTPS client to recover from faulty state!");
+      https_client_reset(client);
+    } else {
+      https_fetch_ctx_cleanup(client, NULL, client->fetches, -1);  // dropping current failed request
+    }
   }
 }
 
@@ -506,10 +514,10 @@ static void https_fetch_ctx_cleanup(https_client_t *client,
   }
   int drop_reply = 0;
   if (curl_result_code < 0) {
-    WLOG_REQ("Request was aborted.");
+    WLOG_REQ("Request was aborted");
     drop_reply = 1;
   } else if (https_fetch_ctx_process_response(client, ctx, curl_result_code) != 0) {
-    ILOG_REQ("Response was faulty, skipping DNS reply.");
+    ILOG_REQ("Response was faulty, skipping DNS reply");
     drop_reply = 1;
   }
   if (drop_reply) {
@@ -545,42 +553,62 @@ static void check_multi_info(https_client_t *c) {
         cur = cur->next;
       }
     }
+    else {
+      ELOG("Unhandled curl message: %d", msg->msg);  // unlikely
+    }
   }
 }
 
 static void sock_cb(struct ev_loop __attribute__((unused)) *loop,
                     struct ev_io *w, int revents) {
   GET_PTR(https_client_t, c, w->data);
+  int ignore = 0;
   CURLMcode code = curl_multi_socket_action(
       c->curlm, w->fd, (revents & EV_READ ? CURL_CSELECT_IN : 0) |
                        (revents & EV_WRITE ? CURL_CSELECT_OUT : 0),
-      &c->still_running);
-  if (code != CURLM_OK) {
+      &ignore);
+  if (code == CURLM_OK) {
+    check_multi_info(c);
+  }
+  else {
     FLOG("curl_multi_socket_action error %d: %s", code, curl_multi_strerror(code));
+    if (code == CURLM_ABORTED_BY_CALLBACK) {
+      WLOG("Resetting HTTPS client to recover from faulty state!");
+      https_client_reset(c);
+    }
   }
-  check_multi_info(c);
 }
 
 static void timer_cb(struct ev_loop __attribute__((unused)) *loop,
                      struct ev_timer *w, int __attribute__((unused)) revents) {
   GET_PTR(https_client_t, c, w->data);
+  int ignore = 0;
   CURLMcode code = curl_multi_socket_action(c->curlm, CURL_SOCKET_TIMEOUT, 0,
-                                            &c->still_running);
+                                            &ignore);
   if (code != CURLM_OK) {
-    FLOG("curl_multi_socket_action error %d: %s", code, curl_multi_strerror(code));
+    ELOG("curl_multi_socket_action error %d: %s", code, curl_multi_strerror(code));
   }
   check_multi_info(c);
 }
 
 static struct ev_io * get_io_event(struct ev_io io_events[], curl_socket_t sock) {
-  for (int i = 0; i < MAX_TOTAL_CONNECTIONS; i++) {
+  for (int i = 0; i < HTTPS_SOCKET_LIMIT; i++) {
     if (io_events[i].fd == sock) {
       return &io_events[i];
     }
   }
   return NULL;
 }
 
+static void dump_io_events(struct ev_io io_events[]) {
+  for (int i = 0; i < HTTPS_SOCKET_LIMIT; i++) {
+    ILOG("IO event #%d: fd=%d, events=%d/%s%s",
+         i+1, io_events[i].fd, io_events[i].events,
+         (io_events[i].events & EV_READ ? "R" : ""),
+         (io_events[i].events & EV_WRITE ? "W" : ""));
+  }
+}
+
 static int multi_sock_cb(CURL *curl, curl_socket_t sock, int what,
                          void *userp, void __attribute__((unused)) *sockp) {
   GET_PTR(https_client_t, c, userp);
@@ -600,7 +628,10 @@ static int multi_sock_cb(CURL *curl, curl_socket_t sock, int what,
   // reserve and start new event on unused slot
   io_event_ptr = get_io_event(c->io_events, 0);
   if (!io_event_ptr) {
-    FLOG("curl needed more event, than max connections: %d", MAX_TOTAL_CONNECTIONS);
+    ELOG("curl needed more IO event handler, than the number of maximum sockets: %d", HTTPS_SOCKET_LIMIT);
+    dump_io_events(c->io_events);
+    logging_flight_recorder_dump();
+    return -1;
   }
   DLOG("Reserved new io event: %p", io_event_ptr);
   // NOLINTNEXTLINE(clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling)
@@ -634,14 +665,15 @@ void https_client_init(https_client_t *c, options_t *opt,
     "Content-Type: " DOH_CONTENT_TYPE);
   c->fetches = NULL;
   c->timer.data = c;
-  for (int i = 0; i < MAX_TOTAL_CONNECTIONS; i++) {
+  for (int i = 0; i < HTTPS_SOCKET_LIMIT; i++) {
     c->io_events[i].data = c;
   }
   c->opt = opt;
   c->stat = stat;
 
   ASSERT_CURL_MULTI_SETOPT(c->curlm, CURLMOPT_PIPELINING, CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX);
-  ASSERT_CURL_MULTI_SETOPT(c->curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, MAX_TOTAL_CONNECTIONS);
+  ASSERT_CURL_MULTI_SETOPT(c->curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, HTTPS_CONNECTION_LIMIT);
+  ASSERT_CURL_MULTI_SETOPT(c->curlm, CURLMOPT_MAX_HOST_CONNECTIONS, HTTPS_CONNECTION_LIMIT);
   ASSERT_CURL_MULTI_SETOPT(c->curlm, CURLMOPT_SOCKETDATA, c);
   ASSERT_CURL_MULTI_SETOPT(c->curlm, CURLMOPT_SOCKETFUNCTION, multi_sock_cb);
   ASSERT_CURL_MULTI_SETOPT(c->curlm, CURLMOPT_TIMERDATA, c);