From 00c9f8de831df2228141469c5a81ea6bcbd1f89b Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 20 Jan 2026 18:21:56 -0500 Subject: [PATCH 01/11] feat: add `soaxreport` tool for ECH testing via SOAX proxies --- soaxreport/soax.go | 113 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 soaxreport/soax.go diff --git a/soaxreport/soax.go b/soaxreport/soax.go new file mode 100644 index 0000000..03079f5 --- /dev/null +++ b/soaxreport/soax.go @@ -0,0 +1,113 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "os" + "strings" +) + +// SoaxConfig holds the credentials and endpoint configuration for the SOAX service. +type SoaxConfig struct { + APIKey string `json:"api_key"` + PackageKey string `json:"package_key"` + PackageID string `json:"package_id"` + ProxyHost string `json:"proxy_host"` + ProxyPort int `json:"proxy_port"` +} + +// SoaxClient provides methods to interact with the SOAX API and generate proxy configurations. +type SoaxClient struct { + Config *SoaxConfig +} + +// NewSoaxClient creates a new SoaxClient by loading configuration from a JSON file. +// If ProxyHost or ProxyPort are missing in the config, default values are used. +func NewSoaxClient(configPath string) (*SoaxClient, error) { + f, err := os.Open(configPath) + if err != nil { + return nil, fmt.Errorf("failed to open config file: %w", err) + } + defer f.Close() + + var cfg SoaxConfig + if err := json.NewDecoder(f).Decode(&cfg); err != nil { + return nil, fmt.Errorf("failed to decode config json: %w", err) + } + if cfg.ProxyHost == "" { + cfg.ProxyHost = "proxy.soax.com" + } + if cfg.ProxyPort == 0 { + cfg.ProxyPort = 5000 + } + + return &SoaxClient{Config: &cfg}, nil +} + +// ListISPs retrieves a list of available ISP operators for the specified country code. +// countryISO should be a 2-letter ISO country code (e.g., "US"). +func (s *SoaxClient) ListISPs(countryISO string) ([]string, error) { + url := fmt.Sprintf("https://api.soax.com/api/get-country-operators?api_key=%s&package_key=%s&country_iso=%s", + s.Config.APIKey, s.Config.PackageKey, strings.ToLower(countryISO)) + + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to fetch ISPs: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API error: status %s, body: %s", resp.Status, string(body)) + } + + var isps []string + if err := json.NewDecoder(resp.Body).Decode(&isps); err != nil { + return nil, fmt.Errorf("failed to decode ISP list: %w", err) + } + return isps, nil +} + +// BuildProxyURL constructs an authenticated HTTPS proxy URL for a specific country and ISP. +// An optional sessionID can be provided for sticky sessions; if empty, a random one is generated. +func (s *SoaxClient) BuildProxyURL(countryISO, ispName, sessionID string) string { + if sessionID == "" { + sessionID = generateRandomString(10) + } + + ispName = url.QueryEscape(strings.ToLower(ispName)) + countryISO = strings.ToLower(countryISO) + + proxyUser := fmt.Sprintf("package-%s-country-%s-isp-%s-sessionid-%s-sessionlength-300", + s.Config.PackageID, countryISO, ispName, sessionID) + + return fmt.Sprintf("https://%s:%s@%s:%d", + proxyUser, s.Config.PackageKey, s.Config.ProxyHost, s.Config.ProxyPort) +} + +func generateRandomString(n int) string { + const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} From 32ed53af41191c3bb7ade7080de32197d483ee9d Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Wed, 21 Jan 2026 16:43:25 -0500 Subject: [PATCH 02/11] Refactor SOAX client into `internal/soax` --- {soaxreport => internal/soax}/soax.go | 37 +++++++++++++++------------ 1 file changed, 21 insertions(+), 16 deletions(-) rename {soaxreport => internal/soax}/soax.go (76%) diff --git a/soaxreport/soax.go b/internal/soax/soax.go similarity index 76% rename from soaxreport/soax.go rename to internal/soax/soax.go index 03079f5..264ff81 100644 --- a/soaxreport/soax.go +++ b/internal/soax/soax.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package soax import ( "encoding/json" @@ -25,8 +25,8 @@ import ( "strings" ) -// SoaxConfig holds the credentials and endpoint configuration for the SOAX service. -type SoaxConfig struct { +// Config holds the credentials and endpoint configuration for the SOAX service. +type Config struct { APIKey string `json:"api_key"` PackageKey string `json:"package_key"` PackageID string `json:"package_id"` @@ -34,21 +34,21 @@ type SoaxConfig struct { ProxyPort int `json:"proxy_port"` } -// SoaxClient provides methods to interact with the SOAX API and generate proxy configurations. -type SoaxClient struct { - Config *SoaxConfig +// Client provides methods to interact with the SOAX API and generate proxy configurations. +type Client struct { + cfg *Config } -// NewSoaxClient creates a new SoaxClient by loading configuration from a JSON file. +// LoadConfig reads the SOAX configuration from a JSON file. // If ProxyHost or ProxyPort are missing in the config, default values are used. -func NewSoaxClient(configPath string) (*SoaxClient, error) { - f, err := os.Open(configPath) +func LoadConfig(path string) (*Config, error) { + f, err := os.Open(path) if err != nil { return nil, fmt.Errorf("failed to open config file: %w", err) } defer f.Close() - var cfg SoaxConfig + var cfg Config if err := json.NewDecoder(f).Decode(&cfg); err != nil { return nil, fmt.Errorf("failed to decode config json: %w", err) } @@ -59,14 +59,19 @@ func NewSoaxClient(configPath string) (*SoaxClient, error) { cfg.ProxyPort = 5000 } - return &SoaxClient{Config: &cfg}, nil + return &cfg, nil +} + +// NewClient creates a new SOAX Client with the given configuration. +func NewClient(cfg *Config) *Client { + return &Client{cfg: cfg} } // ListISPs retrieves a list of available ISP operators for the specified country code. // countryISO should be a 2-letter ISO country code (e.g., "US"). -func (s *SoaxClient) ListISPs(countryISO string) ([]string, error) { +func (c *Client) ListISPs(countryISO string) ([]string, error) { url := fmt.Sprintf("https://api.soax.com/api/get-country-operators?api_key=%s&package_key=%s&country_iso=%s", - s.Config.APIKey, s.Config.PackageKey, strings.ToLower(countryISO)) + c.cfg.APIKey, c.cfg.PackageKey, strings.ToLower(countryISO)) resp, err := http.Get(url) if err != nil { @@ -88,7 +93,7 @@ func (s *SoaxClient) ListISPs(countryISO string) ([]string, error) { // BuildProxyURL constructs an authenticated HTTPS proxy URL for a specific country and ISP. // An optional sessionID can be provided for sticky sessions; if empty, a random one is generated. -func (s *SoaxClient) BuildProxyURL(countryISO, ispName, sessionID string) string { +func (c *Client) BuildProxyURL(countryISO, ispName, sessionID string) string { if sessionID == "" { sessionID = generateRandomString(10) } @@ -97,10 +102,10 @@ func (s *SoaxClient) BuildProxyURL(countryISO, ispName, sessionID string) string countryISO = strings.ToLower(countryISO) proxyUser := fmt.Sprintf("package-%s-country-%s-isp-%s-sessionid-%s-sessionlength-300", - s.Config.PackageID, countryISO, ispName, sessionID) + c.cfg.PackageID, countryISO, ispName, sessionID) return fmt.Sprintf("https://%s:%s@%s:%d", - proxyUser, s.Config.PackageKey, s.Config.ProxyHost, s.Config.ProxyPort) + proxyUser, c.cfg.PackageKey, c.cfg.ProxyHost, c.cfg.ProxyPort) } func generateRandomString(n int) string { From 1972fe790b9887a9c156d61dee93585d2a24da44 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Wed, 21 Jan 2026 18:07:57 -0500 Subject: [PATCH 03/11] Implement a reusable internal/curl package --- internal/curl/exit_codes.go | 110 +++++++++++++++++++++++++ internal/curl/runner.go | 155 ++++++++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 internal/curl/exit_codes.go create mode 100644 internal/curl/runner.go diff --git a/internal/curl/exit_codes.go b/internal/curl/exit_codes.go new file mode 100644 index 0000000..a5e53e9 --- /dev/null +++ b/internal/curl/exit_codes.go @@ -0,0 +1,110 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package curl + +// ExitCodeName returns the human readable name for a curl exit code. +func ExitCodeName(code int) string { + if name, ok := exitCodeNames[code]; ok { + return name + } + return "UNKNOWN_ERROR" +} + +var exitCodeNames = map[int]string{ + 1: "CURLE_UNSUPPORTED_PROTOCOL", + 2: "CURLE_FAILED_INIT", + 3: "CURLE_URL_MALFORMAT", + 4: "CURLE_NOT_BUILT_IN", + 5: "CURLE_COULDNT_RESOLVE_PROXY", + 6: "CURLE_COULDNT_RESOLVE_HOST", + 7: "CURLE_COULDNT_CONNECT", + 8: "CURLE_WEIRD_SERVER_REPLY", + 9: "CURLE_REMOTE_ACCESS_DENIED", + 11: "CURLE_FTP_WEIRD_PASV_REPLY", + 13: "CURLE_FTP_WEIRD_227_FORMAT", + 14: "CURLE_FTP_CANT_GET_HOST", + 15: "CURLE_FTP_CANT_RECONNECT", + 17: "CURLE_FTP_COULDNT_SET_TYPE", + 18: "CURLE_PARTIAL_FILE", + 19: "CURLE_FTP_COULDNT_RETR_FILE", + 21: "CURLE_QUOTE_ERROR", + 22: "CURLE_HTTP_RETURNED_ERROR", + 23: "CURLE_WRITE_ERROR", + 25: "CURLE_UPLOAD_FAILED", + 26: "CURLE_READ_ERROR", + 27: "CURLE_OUT_OF_MEMORY", + 28: "CURLE_OPERATION_TIMEDOUT", + 30: "CURLE_FTP_PORT_FAILED", + 31: "CURLE_FTP_COULDNT_USE_REST", + 33: "CURLE_RANGE_ERROR", + 34: "CURLE_HTTP_POST_ERROR", + 35: "CURLE_SSL_CONNECT_ERROR", + 36: "CURLE_BAD_DOWNLOAD_RESUME", + 37: "CURLE_FILE_COULDNT_READ_FILE", + 38: "CURLE_LDAP_CANNOT_BIND", + 39: "CURLE_LDAP_SEARCH_FAILED", + 41: "CURLE_FUNCTION_NOT_FOUND", + 42: "CURLE_ABORTED_BY_CALLBACK", + 43: "CURLE_BAD_FUNCTION_ARGUMENT", + 45: "CURLE_INTERFACE_FAILED", + 47: "CURLE_TOO_MANY_REDIRECTS", + 48: "CURLE_UNKNOWN_OPTION", + 49: "CURLE_TELNET_OPTION_SYNTAX", + 51: "CURLE_PEER_FAILED_VERIFICATION", + 52: "CURLE_GOT_NOTHING", + 53: "CURLE_SSL_ENGINE_NOTFOUND", + 54: "CURLE_SSL_ENGINE_SETFAILED", + 55: "CURLE_SEND_ERROR", + 56: "CURLE_RECV_ERROR", + 58: "CURLE_SSL_CERTPROBLEM", + 59: "CURLE_SSL_CIPHER", + 60: "CURLE_SSL_CACERT", + 61: "CURLE_BAD_CONTENT_ENCODING", + 62: "CURLE_LDAP_INVALID_URL", + 63: "CURLE_FILESIZE_EXCEEDED", + 64: "CURLE_USE_SSL_FAILED", + 65: "CURLE_SEND_FAIL_REWIND", + 66: "CURLE_SSL_ENGINE_INITFAILED", + 67: "CURLE_LOGIN_DENIED", + 68: "CURLE_TFTP_NOTFOUND", + 69: "CURLE_TFTP_PERM", + 70: "CURLE_REMOTE_DISK_FULL", + 71: "CURLE_TFTP_ILLEGAL", + 72: "CURLE_TFTP_UNKNOWNID", + 73: "CURLE_REMOTE_FILE_EXISTS", + 74: "CURLE_TFTP_NOSUCHUSER", + 75: "CURLE_CONV_FAILED", + 76: "CURLE_CONV_REQD", + 77: "CURLE_SSL_CACERT_BADFILE", + 78: "CURLE_REMOTE_FILE_NOT_FOUND", + 79: "CURLE_SSH", + 80: "CURLE_SSL_SHUTDOWN_FAILED", + 81: "CURLE_AGAIN", + 82: "CURLE_SSL_CRL_BADFILE", + 83: "CURLE_SSL_ISSUER_ERROR", + 84: "CURLE_FTP_PRET_FAILED", + 85: "CURLE_RTSP_CSEQ_ERROR", + 86: "CURLE_RTSP_SESSION_ERROR", + 87: "CURLE_FTP_BAD_FILE_LIST", + 88: "CURLE_CHUNK_FAILED", + 89: "CURLE_NO_CONNECTION_AVAILABLE", + 90: "CURLE_SSL_PINNEDPUBKEYNOTMATCH", + 91: "CURLE_SSL_INVALIDCERTSTATUS", + 92: "CURLE_HTTP2_STREAM", + 93: "CURLE_RECURSIVE_API_CALL", + 94: "CURLE_AUTH_ERROR", + 95: "CURLE_HTTP3", + 96: "CURLE_QUIC_CONNECT_ERROR", +} diff --git a/internal/curl/runner.go b/internal/curl/runner.go new file mode 100644 index 0000000..9747f53 --- /dev/null +++ b/internal/curl/runner.go @@ -0,0 +1,155 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package curl + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +// Runner handles the execution of a specific curl binary. +// It manages environment setup (e.g., LD_LIBRARY_PATH) required for custom builds. +type Runner struct { + curlPath string + libPath string +} + +// Args defines the execution parameters for a single curl run. +type Args struct { + // Proxy specifies the proxy URL to use. + // Corresponds to the "--proxy" flag. + Proxy string + + // ProxyHeaders specifies custom headers to send to the proxy. + // Each string should be in "Key: Value" format. + // Corresponds to the "--proxy-header" flag. + ProxyHeaders []string + + // ECH specifies the Encrypted ClientHello mode. + // Corresponds to the "--ech" flag. + // Use ECHGrease, ECHTrue, ECHFalse constants. + // If empty (ECHNone), the flag is omitted. + ECH ECHMode + + // Verbose enables verbose output. + // If true, adds "-v". + // If false, adds "-s" (silent mode) by default to keep output clean. + Verbose bool + + // Timeout sets the maximum time allowed for the transfer. + // Corresponds to the "--max-time" flag. + // If 0, no timeout is set. + Timeout time.Duration +} + +// ECHMode defines the available Encrypted ClientHello modes for curl. +type ECHMode string + +const ( + // ECHGrease enables ECH GREASE mode ("--ech grease"). + ECHGrease ECHMode = "grease" + // ECHTrue enables ECH ("--ech true"). + ECHTrue ECHMode = "true" + // ECHFalse disables ECH ("--ech false"). + ECHFalse ECHMode = "false" + // ECHNone indicates that the --ech flag should not be sent. + ECHNone ECHMode = "" +) + +// Result represents the raw outcome of a curl execution. +type Result struct { + // ExitCode is the exit status of the curl process. + // 0 indicates success. See ExitCodeName for error name mapping. + ExitCode int + + // Stdout contains the standard output of the curl command. + Stdout string + + // Stderr contains the standard error of the curl command. + // In verbose mode, this contains debug information and headers. + Stderr string +} + +// NewRunner creates a new Runner for the specified curl binary. +// It automatically detects the associated library path (bin/curl -> lib/) +// to ensure shared libraries are found. +func NewRunner(curlPath string) *Runner { + r := &Runner{curlPath: curlPath} + + binDir := filepath.Dir(curlPath) + libDir := filepath.Join(filepath.Dir(binDir), "lib") + if libStat, err := os.Stat(libDir); err == nil && libStat.IsDir() { + r.libPath = libDir + } + + return r +} + +// Run executes curl with the provided arguments and returns the result. +func (r *Runner) Run(url string, args Args) (*Result, error) { + var cmdArgs []string + + if args.Verbose { + cmdArgs = append(cmdArgs, "-v") + } else { + cmdArgs = append(cmdArgs, "-s") + } + + if args.Timeout > 0 { + cmdArgs = append(cmdArgs, "--max-time", strconv.FormatFloat(args.Timeout.Seconds(), 'f', -1, 64)) + } + + if args.Proxy != "" { + cmdArgs = append(cmdArgs, "--proxy", args.Proxy) + } + + for _, h := range args.ProxyHeaders { + cmdArgs = append(cmdArgs, "--proxy-header", h) + } + + if args.ECH != ECHNone { + cmdArgs = append(cmdArgs, "--ech", string(args.ECH)) + } + + cmdArgs = append(cmdArgs, url) + cmd := exec.Command(r.curlPath, cmdArgs...) + if r.libPath != "" { + cmd.Env = append(os.Environ(), "LD_LIBRARY_PATH="+r.libPath) + } + + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + result := &Result{} + err := cmd.Run() + result.Stdout = stdout.String() + result.Stderr = stderr.String() + + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + result.ExitCode = exitErr.ExitCode() + } else { + return result, fmt.Errorf("failed to execute curl: %w", err) + } + } + + return result, nil +} From ac756fbc7dbdc6f1df0ae2214a620969da23d84c Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 27 Jan 2026 16:22:12 -0500 Subject: [PATCH 04/11] implement performance stats collection via `curl -w` --- internal/curl/exit_codes.go | 3 ++ internal/curl/runner.go | 15 ++++++ internal/curl/stats.go | 101 ++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 internal/curl/stats.go diff --git a/internal/curl/exit_codes.go b/internal/curl/exit_codes.go index a5e53e9..e558665 100644 --- a/internal/curl/exit_codes.go +++ b/internal/curl/exit_codes.go @@ -16,6 +16,9 @@ package curl // ExitCodeName returns the human readable name for a curl exit code. func ExitCodeName(code int) string { + if code == 0 { + return "OK" + } if name, ok := exitCodeNames[code]; ok { return name } diff --git a/internal/curl/runner.go b/internal/curl/runner.go index 9747f53..8fb7df1 100644 --- a/internal/curl/runner.go +++ b/internal/curl/runner.go @@ -57,6 +57,10 @@ type Args struct { // Corresponds to the "--max-time" flag. // If 0, no timeout is set. Timeout time.Duration + + // MeasureStats enables capturing performance metrics using curl's -w flag. + // If true, Stats will be populated in the Result. + MeasureStats bool } // ECHMode defines the available Encrypted ClientHello modes for curl. @@ -85,6 +89,9 @@ type Result struct { // Stderr contains the standard error of the curl command. // In verbose mode, this contains debug information and headers. Stderr string + + // Stats contains performance metrics if MeasureStats was enabled. + Stats Stats } // NewRunner creates a new Runner for the specified curl binary. @@ -128,6 +135,10 @@ func (r *Runner) Run(url string, args Args) (*Result, error) { cmdArgs = append(cmdArgs, "--ech", string(args.ECH)) } + if args.MeasureStats { + cmdArgs = append(cmdArgs, "-w", statsFormat) + } + cmdArgs = append(cmdArgs, url) cmd := exec.Command(r.curlPath, cmdArgs...) if r.libPath != "" { @@ -143,6 +154,10 @@ func (r *Runner) Run(url string, args Args) (*Result, error) { result.Stdout = stdout.String() result.Stderr = stderr.String() + if args.MeasureStats { + result.Stats = parseStats(result.Stdout) + } + if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { result.ExitCode = exitErr.ExitCode() diff --git a/internal/curl/stats.go b/internal/curl/stats.go new file mode 100644 index 0000000..a450c55 --- /dev/null +++ b/internal/curl/stats.go @@ -0,0 +1,101 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package curl + +import ( + "strconv" + "strings" + "time" +) + +// Stats captures timing and status metrics from a curl execution. +type Stats struct { + // DNSLookupTimestamp is the cumulative time from the start until the name + // lookup is completed (time_namelookup). + DNSLookupTimestamp time.Duration + + // TCPConnectTimestamp is the cumulative time from the start until the TCP + // connection is completed (time_connect). + TCPConnectTimestamp time.Duration + + // TLSConnectTimestamp is the cumulative time from the start until the + // SSL/TLS handshake is completed (time_appconnect). + TLSConnectTimestamp time.Duration + + // ServerResponseTimestamp is the cumulative time from the start until the + // first byte is received (time_starttransfer). + ServerResponseTimestamp time.Duration + + // TotalTimeTimestamp is the total time from the start until the operation is + // fully completed (time_total). + TotalTimeTimestamp time.Duration + + // HTTPStatus is the HTTP response code (http_code). + HTTPStatus int +} + +const ( + // statsPrefix is the delimiter used to identify the statistics block in the output. + statsPrefix = "\n|||CURL_STATS|||\t" + + // statsFormat is the format string passed to curl's -w flag. + statsFormat = statsPrefix + + "dnslookup:%{time_namelookup}," + + "tcpconnect:%{time_connect}," + + "tlsconnect:%{time_appconnect}," + + "servertime:%{time_starttransfer}," + + "total:%{time_total}," + + "httpstatus:%{http_code}" +) + +// parseStats extracts Stats from the curl output by looking for the statsPrefix. +func parseStats(stdout string) Stats { + var s Stats + idx := strings.LastIndex(stdout, statsPrefix) + if idx == -1 { + return s + } + + raw := strings.TrimSpace(stdout[idx+len(statsPrefix):]) + for part := range strings.SplitSeq(raw, ",") { + kv := strings.Split(part, ":") + if len(kv) != 2 { + continue + } + + key, val := strings.TrimSpace(kv[0]), strings.TrimSpace(kv[1]) + switch key { + case "dnslookup": + s.DNSLookupTimestamp = parseDuration(val) + case "tcpconnect": + s.TCPConnectTimestamp = parseDuration(val) + case "tlsconnect": + s.TLSConnectTimestamp = parseDuration(val) + case "servertime": + s.ServerResponseTimestamp = parseDuration(val) + case "total": + s.TotalTimeTimestamp = parseDuration(val) + case "httpstatus": + s.HTTPStatus, _ = strconv.Atoi(val) + } + } + return s +} + +// parseDuration converts a seconds-based float string to time.Duration. +func parseDuration(s string) time.Duration { + f, _ := strconv.ParseFloat(s, 64) + return time.Duration(f * float64(time.Second)) +} From 55170b9328372d26d55fec46d1311c793cb11f31 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 27 Jan 2026 16:45:09 -0500 Subject: [PATCH 05/11] implement ECH testing with SOAX proxy --- soaxreport/main.go | 247 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 soaxreport/main.go diff --git a/soaxreport/main.go b/soaxreport/main.go new file mode 100644 index 0000000..2a786f2 --- /dev/null +++ b/soaxreport/main.go @@ -0,0 +1,247 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "encoding/csv" + "flag" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/Jigsaw-Code/ech-research/internal/curl" + "github.com/Jigsaw-Code/ech-research/internal/soax" + "github.com/Jigsaw-Code/ech-research/internal/workspace" +) + +type TestResult struct { + Domain string + Country string + ISP string + ASN string + ExitNodeIP string + ECHGrease bool + Error string + CurlExitCode int + CurlErrorName string + DNSLookup time.Duration + TCPConnection time.Duration + TLSHandshake time.Duration + ServerTime time.Duration + TotalTime time.Duration + HTTPStatus int +} + +func runSoaxTest(runner *curl.Runner, domain string, country string, isp string, proxyURL string, echGrease bool, maxTime time.Duration, verbose bool) TestResult { + result := TestResult{ + Domain: domain, + Country: country, + ISP: isp, + ECHGrease: echGrease, + } + + echMode := curl.ECHFalse + if echGrease { + echMode = curl.ECHGrease + } + + url := "https://" + domain + res, err := runner.Run(url, curl.Args{ + Proxy: proxyURL, + ProxyHeaders: []string{"Respond-With: ip,isp,asn"}, + ECH: echMode, + Timeout: maxTime, + Verbose: verbose, + MeasureStats: true, + }) + + result.CurlExitCode = res.ExitCode + result.CurlErrorName = curl.ExitCodeName(res.ExitCode) + result.HTTPStatus = res.Stats.HTTPStatus + result.DNSLookup = res.Stats.DNSLookupTimestamp + result.TCPConnection = res.Stats.TCPConnectTimestamp + result.TLSHandshake = res.Stats.TLSConnectTimestamp + result.ServerTime = res.Stats.ServerResponseTimestamp + result.TotalTime = res.Stats.TotalTimeTimestamp + + if err != nil { + result.Error = err.Error() + } + + // Parse metadata from Stderr (SOAX specific headers in CONNECT response) + for line := range strings.SplitSeq(res.Stderr, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "< Node-") { + continue + } + kv := strings.SplitN(strings.TrimPrefix(line, "< Node-"), ":", 2) + if len(kv) != 2 { + continue + } + key := strings.ToLower(strings.TrimSpace(kv[0])) + val := strings.TrimSpace(kv[1]) + switch key { + case "asn": + result.ASN = val + case "ip": + result.ExitNodeIP = val + case "isp": + result.ISP = val + } + } + + return result +} + +func loadCountries(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + var countries []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "" && !strings.HasPrefix(line, "#") { + countries = append(countries, line) + } + } + return countries, scanner.Err() +} + +func main() { + var ( + workspaceFlag = flag.String("workspace", "./workspace", "Directory to store intermediate files") + soaxConfigFlag = flag.String("soax", "", "Path to SOAX config JSON") + countriesFlag = flag.String("countries", "", "Path to file containing ISO country codes") + targetDomainFlag = flag.String("targetDomain", "www.google.com", "Target domain to test") + verboseFlag = flag.Bool("verbose", false, "Enable verbose logging") + maxTimeFlag = flag.Duration("maxTime", 30*time.Second, "Maximum time per curl request") + curlPathFlag = flag.String("curl", "", "Path to the ECH-enabled curl binary") + ) + flag.Parse() + + if *verboseFlag { + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))) + } else { + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))) + } + + // Set up workspace directory + workspaceDir := workspace.EnsureWorkspace(*workspaceFlag) + + // Determine curl binary path + curlPath := *curlPathFlag + if curlPath == "" { + curlPath = filepath.Join(workspaceDir, "output", "bin", "curl") + } + runner := curl.NewRunner(curlPath) + + // Load SOAX config + soaxConfigPath := *soaxConfigFlag + if soaxConfigPath == "" { + soaxConfigPath = filepath.Join(workspaceDir, "soax", "cred.json") + } + cfg, err := soax.LoadConfig(soaxConfigPath) + if err != nil { + slog.Error("Failed to load SOAX config", "path", soaxConfigPath, "error", err) + os.Exit(1) + } + client := soax.NewClient(cfg) + + // Load countries + if *countriesFlag == "" { + slog.Error("The --countries flag is required") + os.Exit(1) + } + countries, err := loadCountries(*countriesFlag) + if err != nil { + slog.Error("Failed to load countries list", "path", *countriesFlag, "error", err) + os.Exit(1) + } + + // Create output CSV file + outputFilename := filepath.Join(workspaceDir, fmt.Sprintf("soax-results-countries%d.csv", len(countries))) + outputFile, err := os.Create(outputFilename) + if err != nil { + slog.Error("Failed to create output CSV file", "path", outputFilename, "error", err) + os.Exit(1) + } + defer outputFile.Close() + + csvWriter := csv.NewWriter(outputFile) + defer csvWriter.Flush() + + header := []string{ + "domain", "country", "isp", "asn", "exit_node_ip", "ech_grease", "error", + "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", + "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", + } + if err := csvWriter.Write(header); err != nil { + slog.Error("Failed to write CSV header", "error", err) + os.Exit(1) + } + + domain := *targetDomainFlag + + for _, country := range countries { + slog.Info("Processing country", "country", country) + + isps, err := client.ListISPs(country) + if err != nil { + slog.Error("Failed to fetch ISPs", "country", country, "error", err) + continue + } + + for _, isp := range isps { + proxyURL := client.BuildProxyURL(country, isp, "") + + // Test ECH False + slog.Info("Testing ISP", "country", country, "isp", isp, "ech_grease", false) + resFalse := runSoaxTest(runner, domain, country, isp, proxyURL, false, *maxTimeFlag, *verboseFlag) + + // Test ECH Grease + slog.Info("Testing ISP", "country", country, "isp", isp, "ech_grease", true) + resGrease := runSoaxTest(runner, domain, country, isp, proxyURL, true, *maxTimeFlag, *verboseFlag) + + results := []TestResult{resFalse, resGrease} + for _, r := range results { + record := []string{ + r.Domain, r.Country, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, + strconv.Itoa(r.CurlExitCode), r.CurlErrorName, + strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), + strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), + strconv.FormatInt(r.TLSHandshake.Milliseconds(), 10), + strconv.FormatInt(r.ServerTime.Milliseconds(), 10), + strconv.FormatInt(r.TotalTime.Milliseconds(), 10), + strconv.Itoa(r.HTTPStatus), + } + if err := csvWriter.Write(record); err != nil { + slog.Error("Failed to write record to CSV", "error", err) + } + } + csvWriter.Flush() + } + } + + slog.Info("Done. Results saved to", "path", outputFilename) +} From 33397dd3c70186d0d212d3881cbf54bf26ea37e2 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Thu, 29 Jan 2026 00:03:38 -0500 Subject: [PATCH 06/11] Implement concurrent SOAX ECH testing --- soaxreport/README.md | 85 +++++++++++++++++++++++++++++ soaxreport/main.go | 126 ++++++++++++++++++++++++++++--------------- 2 files changed, 168 insertions(+), 43 deletions(-) create mode 100644 soaxreport/README.md diff --git a/soaxreport/README.md b/soaxreport/README.md new file mode 100644 index 0000000..a83b4f0 --- /dev/null +++ b/soaxreport/README.md @@ -0,0 +1,85 @@ +# SOAX ECH GREASE Report Generation + +This tool tests ECH GREASE compatibility by issuing requests via SOAX proxies. +It iterates through a list of countries and ISPs, running tests with and without +ECH GREASE to simulate diverse network vantage points. + +## Requirements + +You need to build the ECH-enabled `curl` and place it in the workspace directory. See [instructions](../curl/README.md). + +You also need a SOAX configuration file (`soax/cred.json` in the workspace) and a list of ISO country codes. + +### Configuration File Examples + +**SOAX Credentials (`soax/cred.json`)** + +The SOAX configuration file should be a JSON file with the following structure: + +```json +{ + "api_key": "YOUR_API_KEY", + "package_key": "YOUR_PACKAGE_KEY", + "package_id": "YOUR_PACKAGE_ID", + "proxy_host": "proxy.soax.com", + "proxy_port": 5000 +} +``` + +**Country List (`countries.txt`)** + +The countries file should contain a list of 2-letter ISO country codes, one per line. Lines starting with `#` are ignored. + +```text +US +GB +DE +# Add more countries as needed +JP +``` + +## Running + +To run the tool, use the `go run` command from the project root directory: + +```sh +go run ./soaxreport --countries workspace/countries.txt --targetDomain www.google.com +``` + +This will: + +1. Load the SOAX credentials (`./workspace/soax/cred.json` by default) and country list. +2. For each country, fetch the list of available ISPs. +3. For each ISP, issue requests to the target domain via a SOAX proxy, once with ECH GREASE and once without. +4. Save the results to `./workspace/soax-results--countries.csv`. + +### Parameters + +* `-workspace `: Directory to store intermediate files. Defaults to `./workspace`. +* `-soax `: Path to SOAX config JSON. Defaults to `./workspace/soax/cred.json`. +* `-countries `: Path to file containing ISO country codes (required). +* `-targetDomain `: Target domain to test. Defaults to `www.google.com`. +* `-parallelism `: Maximum number of parallel requests. Defaults to 10. +* `-verbose`: Enable verbose logging. +* `-maxTime `: Maximum time per curl request. Defaults to `30s`. +* `-curl `: Path to the ECH-enabled curl binary. Defaults to `./workspace/output/bin/curl`. + +### Output Format + +The tool generates a CSV file (`workspace/soax-results--countries.csv`) with the following columns: + +* `domain`: The domain that was tested. +* `country`: The country code of the proxy used. +* `isp`: The ISP name of the proxy used. +* `asn`: The ASN of the proxy exit node. +* `exit_node_ip`: The IP address of the proxy exit node. +* `ech_grease`: `true` if ECH GREASE was enabled for the request, `false` otherwise. +* `error`: Any error that occurred during the request. +* `curl_exit_code`: The exit code returned by the `curl` command. +* `curl_error_name`: The human-readable name corresponding to the `curl` exit code. +* `dns_lookup_ms`: The duration of the DNS lookup. +* `tcp_connection_ms`: The duration of the TCP connection. +* `tls_handshake_ms`: The duration of the TLS handshake. +* `server_time_ms`: The time from the end of the TLS handshake to the first byte of the response. +* `total_time_ms`: The total duration of the request. +* `http_status`: The HTTP status code of the response. diff --git a/soaxreport/main.go b/soaxreport/main.go index 2a786f2..e0e52a3 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -16,6 +16,7 @@ package main import ( "bufio" + "context" "encoding/csv" "flag" "fmt" @@ -24,11 +25,13 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" "github.com/Jigsaw-Code/ech-research/internal/curl" "github.com/Jigsaw-Code/ech-research/internal/soax" "github.com/Jigsaw-Code/ech-research/internal/workspace" + "golang.org/x/sync/semaphore" ) type TestResult struct { @@ -49,7 +52,15 @@ type TestResult struct { HTTPStatus int } -func runSoaxTest(runner *curl.Runner, domain string, country string, isp string, proxyURL string, echGrease bool, maxTime time.Duration, verbose bool) TestResult { +func runSoaxTest( + runner *curl.Runner, + domain string, + country string, + isp string, + proxyURL string, + echGrease bool, + maxTime time.Duration, +) TestResult { result := TestResult{ Domain: domain, Country: country, @@ -68,7 +79,7 @@ func runSoaxTest(runner *curl.Runner, domain string, country string, isp string, ProxyHeaders: []string{"Respond-With: ip,isp,asn"}, ECH: echMode, Timeout: maxTime, - Verbose: verbose, + Verbose: true, // Required to capture response headers MeasureStats: true, }) @@ -103,7 +114,7 @@ func runSoaxTest(runner *curl.Runner, domain string, country string, isp string, case "ip": result.ExitNodeIP = val case "isp": - result.ISP = val + result.ISP += " (" + val + ")" } } @@ -137,6 +148,7 @@ func main() { verboseFlag = flag.Bool("verbose", false, "Enable verbose logging") maxTimeFlag = flag.Duration("maxTime", 30*time.Second, "Maximum time per curl request") curlPathFlag = flag.String("curl", "", "Path to the ECH-enabled curl binary") + parallelismFlag = flag.Int("parallelism", 10, "Maximum number of parallel requests") ) flag.Parse() @@ -180,7 +192,8 @@ func main() { } // Create output CSV file - outputFilename := filepath.Join(workspaceDir, fmt.Sprintf("soax-results-countries%d.csv", len(countries))) + sanitizedDomain := strings.ReplaceAll(*targetDomainFlag, ".", "_") + outputFilename := filepath.Join(workspaceDir, fmt.Sprintf("soax-results-%s-countries%d.csv", sanitizedDomain, len(countries))) outputFile, err := os.Create(outputFilename) if err != nil { slog.Error("Failed to create output CSV file", "path", outputFilename, "error", err) @@ -188,23 +201,46 @@ func main() { } defer outputFile.Close() - csvWriter := csv.NewWriter(outputFile) - defer csvWriter.Flush() + resultsCh := make(chan TestResult, 2*len(countries)*(*parallelismFlag)) - header := []string{ - "domain", "country", "isp", "asn", "exit_node_ip", "ech_grease", "error", - "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", - "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", - } - if err := csvWriter.Write(header); err != nil { - slog.Error("Failed to write CSV header", "error", err) - os.Exit(1) - } + var csvWg sync.WaitGroup + csvWg.Add(1) + go func() { + defer csvWg.Done() + csvWriter := csv.NewWriter(outputFile) + defer csvWriter.Flush() - domain := *targetDomainFlag + header := []string{ + "domain", "country", "isp", "asn", "exit_node_ip", "ech_grease", "error", + "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", + "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", + } + if err := csvWriter.Write(header); err != nil { + slog.Error("Failed to write CSV header", "error", err) + } + for r := range resultsCh { + record := []string{ + r.Domain, r.Country, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, + strconv.Itoa(r.CurlExitCode), r.CurlErrorName, + strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), + strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), + strconv.FormatInt(r.TLSHandshake.Milliseconds(), 10), + strconv.FormatInt(r.ServerTime.Milliseconds(), 10), + strconv.FormatInt(r.TotalTime.Milliseconds(), 10), + strconv.Itoa(r.HTTPStatus), + } + if err := csvWriter.Write(record); err != nil { + slog.Error("Failed to write record to CSV", "error", err) + } + } + }() + + domain := *targetDomainFlag + sem := semaphore.NewWeighted(int64(*parallelismFlag)) + var wg sync.WaitGroup for _, country := range countries { - slog.Info("Processing country", "country", country) + slog.Debug("Processing country", "country", country) isps, err := client.ListISPs(country) if err != nil { @@ -213,35 +249,39 @@ func main() { } for _, isp := range isps { - proxyURL := client.BuildProxyURL(country, isp, "") - - // Test ECH False - slog.Info("Testing ISP", "country", country, "isp", isp, "ech_grease", false) - resFalse := runSoaxTest(runner, domain, country, isp, proxyURL, false, *maxTimeFlag, *verboseFlag) - - // Test ECH Grease - slog.Info("Testing ISP", "country", country, "isp", isp, "ech_grease", true) - resGrease := runSoaxTest(runner, domain, country, isp, proxyURL, true, *maxTimeFlag, *verboseFlag) - - results := []TestResult{resFalse, resGrease} - for _, r := range results { - record := []string{ - r.Domain, r.Country, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, - strconv.Itoa(r.CurlExitCode), r.CurlErrorName, - strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), - strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), - strconv.FormatInt(r.TLSHandshake.Milliseconds(), 10), - strconv.FormatInt(r.ServerTime.Milliseconds(), 10), - strconv.FormatInt(r.TotalTime.Milliseconds(), 10), - strconv.Itoa(r.HTTPStatus), - } - if err := csvWriter.Write(record); err != nil { - slog.Error("Failed to write record to CSV", "error", err) - } + wg.Add(2) + + if err := sem.Acquire(context.Background(), 1); err != nil { + slog.Error("Failed to acquire semaphore", "error", err) + wg.Done() + } else { + go func(c, isp string) { + defer sem.Release(1) + defer wg.Done() + proxyURL := client.BuildProxyURL(c, isp, "") + slog.Info("Testing ISP", "country", c, "isp", isp, "ech_grease", false) + resultsCh <- runSoaxTest(runner, domain, c, isp, proxyURL, false, *maxTimeFlag) + }(country, isp) + } + + if err := sem.Acquire(context.Background(), 1); err != nil { + slog.Error("Failed to acquire semaphore", "error", err) + wg.Done() + } else { + go func(c, isp string) { + defer sem.Release(1) + defer wg.Done() + proxyURL := client.BuildProxyURL(c, isp, "") + slog.Info("Testing ISP", "country", c, "isp", isp, "ech_grease", true) + resultsCh <- runSoaxTest(runner, domain, c, isp, proxyURL, true, *maxTimeFlag) + }(country, isp) } - csvWriter.Flush() } } + wg.Wait() + close(resultsCh) + csvWg.Wait() + slog.Info("Done. Results saved to", "path", outputFilename) } From 07a674094ff294db82fbc4d47a4f241fb2abccc3 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Thu, 29 Jan 2026 01:16:38 -0500 Subject: [PATCH 07/11] Refactor to use CSV country list and include country names --- soaxreport/README.md | 19 ++++++----- soaxreport/main.go | 75 +++++++++++++++++++++++++++++--------------- 2 files changed, 60 insertions(+), 34 deletions(-) diff --git a/soaxreport/README.md b/soaxreport/README.md index a83b4f0..c1ccb89 100644 --- a/soaxreport/README.md +++ b/soaxreport/README.md @@ -28,16 +28,18 @@ The SOAX configuration file should be a JSON file with the following structure: **Country List (`countries.txt`)** -The countries file should contain a list of 2-letter ISO country codes, one per line. Lines starting with `#` are ignored. +The countries file should be a CSV file containing country names and their 2-letter ISO codes. Lines starting with `#` are ignored. -```text -US -GB -DE +```csv +"United States",US +"United Kingdom",GB +"Germany",DE # Add more countries as needed -JP +"Virgin Islands, U.S.",VI ``` +You can download a complete list of country codes from [here](https://raw.githubusercontent.com/datasets/country-list/master/data.csv). + ## Running To run the tool, use the `go run` command from the project root directory: @@ -57,7 +59,7 @@ This will: * `-workspace `: Directory to store intermediate files. Defaults to `./workspace`. * `-soax `: Path to SOAX config JSON. Defaults to `./workspace/soax/cred.json`. -* `-countries `: Path to file containing ISO country codes (required). +* `-countries `: Path to CSV file containing country names and ISO codes (required). * `-targetDomain `: Target domain to test. Defaults to `www.google.com`. * `-parallelism `: Maximum number of parallel requests. Defaults to 10. * `-verbose`: Enable verbose logging. @@ -69,7 +71,8 @@ This will: The tool generates a CSV file (`workspace/soax-results--countries.csv`) with the following columns: * `domain`: The domain that was tested. -* `country`: The country code of the proxy used. +* `country_code`: The 2-letter ISO country code. +* `country_name`: The full name of the country. * `isp`: The ISP name of the proxy used. * `asn`: The ASN of the proxy exit node. * `exit_node_ip`: The IP address of the proxy exit node. diff --git a/soaxreport/main.go b/soaxreport/main.go index e0e52a3..4a18faf 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -15,7 +15,6 @@ package main import ( - "bufio" "context" "encoding/csv" "flag" @@ -37,6 +36,7 @@ import ( type TestResult struct { Domain string Country string + CountryName string ISP string ASN string ExitNodeIP string @@ -56,16 +56,18 @@ func runSoaxTest( runner *curl.Runner, domain string, country string, + countryName string, isp string, proxyURL string, echGrease bool, maxTime time.Duration, ) TestResult { result := TestResult{ - Domain: domain, - Country: country, - ISP: isp, - ECHGrease: echGrease, + Domain: domain, + Country: country, + CountryName: countryName, + ISP: isp, + ECHGrease: echGrease, } echMode := curl.ECHFalse @@ -121,22 +123,43 @@ func runSoaxTest( return result } -func loadCountries(path string) ([]string, error) { +type Country struct { + Name string + Code string +} + +func loadCountries(path string) ([]Country, error) { f, err := os.Open(path) if err != nil { return nil, err } defer f.Close() - var countries []string - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line != "" && !strings.HasPrefix(line, "#") { - countries = append(countries, line) + var countries []Country + reader := csv.NewReader(f) + reader.Comment = '#' // Support skipping lines starting with # + reader.FieldsPerRecord = 2 + + records, err := reader.ReadAll() + if err != nil { + return nil, fmt.Errorf("failed to read countries CSV: %w", err) + } + + for i, record := range records { + name := strings.TrimSpace(record[0]) + code := strings.TrimSpace(record[1]) + + // Skip header row if present + if i == 0 && strings.EqualFold(name, "Name") && strings.EqualFold(code, "Code") { + continue } + + countries = append(countries, Country{ + Name: name, + Code: code, + }) } - return countries, scanner.Err() + return countries, nil } func main() { @@ -211,7 +234,7 @@ func main() { defer csvWriter.Flush() header := []string{ - "domain", "country", "isp", "asn", "exit_node_ip", "ech_grease", "error", + "domain", "country_code", "country_name", "isp", "asn", "exit_node_ip", "ech_grease", "error", "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", } @@ -221,7 +244,7 @@ func main() { for r := range resultsCh { record := []string{ - r.Domain, r.Country, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, + r.Domain, r.Country, r.CountryName, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, strconv.Itoa(r.CurlExitCode), r.CurlErrorName, strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), @@ -240,11 +263,11 @@ func main() { sem := semaphore.NewWeighted(int64(*parallelismFlag)) var wg sync.WaitGroup for _, country := range countries { - slog.Debug("Processing country", "country", country) + slog.Debug("Processing country", "name", country.Name, "code", country.Code) - isps, err := client.ListISPs(country) + isps, err := client.ListISPs(country.Code) if err != nil { - slog.Error("Failed to fetch ISPs", "country", country, "error", err) + slog.Error("Failed to fetch ISPs", "country", country.Code, "error", err) continue } @@ -255,12 +278,12 @@ func main() { slog.Error("Failed to acquire semaphore", "error", err) wg.Done() } else { - go func(c, isp string) { + go func(c Country, isp string) { defer sem.Release(1) defer wg.Done() - proxyURL := client.BuildProxyURL(c, isp, "") - slog.Info("Testing ISP", "country", c, "isp", isp, "ech_grease", false) - resultsCh <- runSoaxTest(runner, domain, c, isp, proxyURL, false, *maxTimeFlag) + proxyURL := client.BuildProxyURL(c.Code, isp, "") + slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", false) + resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, false, *maxTimeFlag) }(country, isp) } @@ -268,12 +291,12 @@ func main() { slog.Error("Failed to acquire semaphore", "error", err) wg.Done() } else { - go func(c, isp string) { + go func(c Country, isp string) { defer sem.Release(1) defer wg.Done() - proxyURL := client.BuildProxyURL(c, isp, "") - slog.Info("Testing ISP", "country", c, "isp", isp, "ech_grease", true) - resultsCh <- runSoaxTest(runner, domain, c, isp, proxyURL, true, *maxTimeFlag) + proxyURL := client.BuildProxyURL(c.Code, isp, "") + slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", true) + resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, true, *maxTimeFlag) }(country, isp) } } From d7f5a50b533cac587b55cbc16cc07c25bca40d5c Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Thu, 29 Jan 2026 01:31:16 -0500 Subject: [PATCH 08/11] Ensure same exit node for one ISP thru sticky session. --- soaxreport/main.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/soaxreport/main.go b/soaxreport/main.go index 4a18faf..afdb070 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -260,6 +260,7 @@ func main() { }() domain := *targetDomainFlag + runSessionID := time.Now().Format("0102150405") sem := semaphore.NewWeighted(int64(*parallelismFlag)) var wg sync.WaitGroup for _, country := range countries { @@ -271,33 +272,34 @@ func main() { continue } - for _, isp := range isps { + for i, isp := range isps { wg.Add(2) + sessionID := fmt.Sprintf("%s%s%d", runSessionID, country.Code, i) if err := sem.Acquire(context.Background(), 1); err != nil { slog.Error("Failed to acquire semaphore", "error", err) wg.Done() } else { - go func(c Country, isp string) { + go func(c Country, isp, sid string) { defer sem.Release(1) defer wg.Done() - proxyURL := client.BuildProxyURL(c.Code, isp, "") - slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", false) + proxyURL := client.BuildProxyURL(c.Code, isp, sid) + slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", false, "session", sid) resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, false, *maxTimeFlag) - }(country, isp) + }(country, isp, sessionID) } if err := sem.Acquire(context.Background(), 1); err != nil { slog.Error("Failed to acquire semaphore", "error", err) wg.Done() } else { - go func(c Country, isp string) { + go func(c Country, isp, sid string) { defer sem.Release(1) defer wg.Done() - proxyURL := client.BuildProxyURL(c.Code, isp, "") - slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", true) + proxyURL := client.BuildProxyURL(c.Code, isp, sid) + slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", true, "session", sid) resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, true, *maxTimeFlag) - }(country, isp) + }(country, isp, sessionID) } } } From e6fa052dcbdf0ff51bf3c96c0f534a8d81e2459b Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Mon, 2 Feb 2026 16:34:28 -0500 Subject: [PATCH 09/11] Implement atomic progress tracking and simplify test func --- soaxreport/README.md | 4 ++-- soaxreport/main.go | 44 ++++++++++++++++++++------------------------ 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/soaxreport/README.md b/soaxreport/README.md index c1ccb89..71ad5f3 100644 --- a/soaxreport/README.md +++ b/soaxreport/README.md @@ -26,7 +26,7 @@ The SOAX configuration file should be a JSON file with the following structure: } ``` -**Country List (`countries.txt`)** +**Country List (`countries.csv`)** The countries file should be a CSV file containing country names and their 2-letter ISO codes. Lines starting with `#` are ignored. @@ -45,7 +45,7 @@ You can download a complete list of country codes from [here](https://raw.github To run the tool, use the `go run` command from the project root directory: ```sh -go run ./soaxreport --countries workspace/countries.txt --targetDomain www.google.com +go run ./soaxreport --countries workspace/countries.csv --targetDomain www.google.com ``` This will: diff --git a/soaxreport/main.go b/soaxreport/main.go index afdb070..805665a 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -25,6 +25,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/Jigsaw-Code/ech-research/internal/curl" @@ -171,7 +172,7 @@ func main() { verboseFlag = flag.Bool("verbose", false, "Enable verbose logging") maxTimeFlag = flag.Duration("maxTime", 30*time.Second, "Maximum time per curl request") curlPathFlag = flag.String("curl", "", "Path to the ECH-enabled curl binary") - parallelismFlag = flag.Int("parallelism", 10, "Maximum number of parallel requests") + parallelismFlag = flag.Int("parallelism", 16, "Maximum number of parallel requests") ) flag.Parse() @@ -263,6 +264,8 @@ func main() { runSessionID := time.Now().Format("0102150405") sem := semaphore.NewWeighted(int64(*parallelismFlag)) var wg sync.WaitGroup + var total, finished atomic.Int32 + for _, country := range countries { slog.Debug("Processing country", "name", country.Name, "code", country.Code) @@ -272,35 +275,28 @@ func main() { continue } + total.Add(int32(len(isps) * 2)) for i, isp := range isps { wg.Add(2) sessionID := fmt.Sprintf("%s%s%d", runSessionID, country.Code, i) - if err := sem.Acquire(context.Background(), 1); err != nil { - slog.Error("Failed to acquire semaphore", "error", err) - wg.Done() - } else { - go func(c Country, isp, sid string) { - defer sem.Release(1) - defer wg.Done() - proxyURL := client.BuildProxyURL(c.Code, isp, sid) - slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", false, "session", sid) - resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, false, *maxTimeFlag) - }(country, isp, sessionID) + startTest := func(c Country, isp, sid string, ech bool) { + defer wg.Done() + if err := sem.Acquire(context.Background(), 1); err != nil { + slog.Error("Failed to acquire semaphore", "error", err) + return + } + defer sem.Release(1) + + proxyURL := client.BuildProxyURL(c.Code, isp, sid) + slog.Debug("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", ech, "session", sid) + resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, ech, *maxTimeFlag) + progress := fmt.Sprintf("%d/%d", finished.Add(1), total.Load()) + slog.Info("Finished", "country", c.Code, "isp", isp, "progress", progress) } - if err := sem.Acquire(context.Background(), 1); err != nil { - slog.Error("Failed to acquire semaphore", "error", err) - wg.Done() - } else { - go func(c Country, isp, sid string) { - defer sem.Release(1) - defer wg.Done() - proxyURL := client.BuildProxyURL(c.Code, isp, sid) - slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", true, "session", sid) - resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, true, *maxTimeFlag) - }(country, isp, sessionID) - } + go startTest(country, isp, sessionID, false) + go startTest(country, isp, sessionID, true) } } From e978f1e4aac3c4fa82acc6ebe0e8bd797db75f2d Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Mon, 2 Feb 2026 17:32:20 -0500 Subject: [PATCH 10/11] Separate input ISP and exit node ISP header into different columns. --- soaxreport/README.md | 1 + soaxreport/main.go | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/soaxreport/README.md b/soaxreport/README.md index 71ad5f3..76cd4af 100644 --- a/soaxreport/README.md +++ b/soaxreport/README.md @@ -76,6 +76,7 @@ The tool generates a CSV file (`workspace/soax-results--countries.csv * `isp`: The ISP name of the proxy used. * `asn`: The ASN of the proxy exit node. * `exit_node_ip`: The IP address of the proxy exit node. +* `exit_node_isp`: The ISP name reported by the proxy exit node (from headers). * `ech_grease`: `true` if ECH GREASE was enabled for the request, `false` otherwise. * `error`: Any error that occurred during the request. * `curl_exit_code`: The exit code returned by the `curl` command. diff --git a/soaxreport/main.go b/soaxreport/main.go index 805665a..cc08279 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -41,6 +41,7 @@ type TestResult struct { ISP string ASN string ExitNodeIP string + ExitNodeISP string ECHGrease bool Error string CurlExitCode int @@ -117,7 +118,7 @@ func runSoaxTest( case "ip": result.ExitNodeIP = val case "isp": - result.ISP += " (" + val + ")" + result.ExitNodeISP = val } } @@ -235,7 +236,7 @@ func main() { defer csvWriter.Flush() header := []string{ - "domain", "country_code", "country_name", "isp", "asn", "exit_node_ip", "ech_grease", "error", + "domain", "country_code", "country_name", "isp", "asn", "exit_node_ip", "exit_node_isp", "ech_grease", "error", "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", } @@ -245,7 +246,7 @@ func main() { for r := range resultsCh { record := []string{ - r.Domain, r.Country, r.CountryName, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, + r.Domain, r.Country, r.CountryName, r.ISP, r.ASN, r.ExitNodeIP, r.ExitNodeISP, strconv.FormatBool(r.ECHGrease), r.Error, strconv.Itoa(r.CurlExitCode), r.CurlErrorName, strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), From f0557b7c5fc2fda68bd1c29abd40e1cbba61fe11 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Wed, 4 Feb 2026 16:50:06 -0500 Subject: [PATCH 11/11] Update parallelism default value in README for soaxreport --- soaxreport/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soaxreport/README.md b/soaxreport/README.md index 76cd4af..ee54c91 100644 --- a/soaxreport/README.md +++ b/soaxreport/README.md @@ -61,7 +61,7 @@ This will: * `-soax `: Path to SOAX config JSON. Defaults to `./workspace/soax/cred.json`. * `-countries `: Path to CSV file containing country names and ISO codes (required). * `-targetDomain `: Target domain to test. Defaults to `www.google.com`. -* `-parallelism `: Maximum number of parallel requests. Defaults to 10. +* `-parallelism `: Maximum number of parallel requests. Defaults to `16`. * `-verbose`: Enable verbose logging. * `-maxTime `: Maximum time per curl request. Defaults to `30s`. * `-curl `: Path to the ECH-enabled curl binary. Defaults to `./workspace/output/bin/curl`.