diff --git a/internal/curl/exit_codes.go b/internal/curl/exit_codes.go new file mode 100644 index 0000000..e558665 --- /dev/null +++ b/internal/curl/exit_codes.go @@ -0,0 +1,113 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package curl + +// ExitCodeName returns the human readable name for a curl exit code. +func ExitCodeName(code int) string { + if code == 0 { + return "OK" + } + if name, ok := exitCodeNames[code]; ok { + return name + } + return "UNKNOWN_ERROR" +} + +var exitCodeNames = map[int]string{ + 1: "CURLE_UNSUPPORTED_PROTOCOL", + 2: "CURLE_FAILED_INIT", + 3: "CURLE_URL_MALFORMAT", + 4: "CURLE_NOT_BUILT_IN", + 5: "CURLE_COULDNT_RESOLVE_PROXY", + 6: "CURLE_COULDNT_RESOLVE_HOST", + 7: "CURLE_COULDNT_CONNECT", + 8: "CURLE_WEIRD_SERVER_REPLY", + 9: "CURLE_REMOTE_ACCESS_DENIED", + 11: "CURLE_FTP_WEIRD_PASV_REPLY", + 13: "CURLE_FTP_WEIRD_227_FORMAT", + 14: "CURLE_FTP_CANT_GET_HOST", + 15: "CURLE_FTP_CANT_RECONNECT", + 17: "CURLE_FTP_COULDNT_SET_TYPE", + 18: "CURLE_PARTIAL_FILE", + 19: "CURLE_FTP_COULDNT_RETR_FILE", + 21: "CURLE_QUOTE_ERROR", + 22: "CURLE_HTTP_RETURNED_ERROR", + 23: "CURLE_WRITE_ERROR", + 25: "CURLE_UPLOAD_FAILED", + 26: "CURLE_READ_ERROR", + 27: "CURLE_OUT_OF_MEMORY", + 28: "CURLE_OPERATION_TIMEDOUT", + 30: "CURLE_FTP_PORT_FAILED", + 31: "CURLE_FTP_COULDNT_USE_REST", + 33: "CURLE_RANGE_ERROR", + 34: "CURLE_HTTP_POST_ERROR", + 35: "CURLE_SSL_CONNECT_ERROR", + 36: "CURLE_BAD_DOWNLOAD_RESUME", + 37: "CURLE_FILE_COULDNT_READ_FILE", + 38: "CURLE_LDAP_CANNOT_BIND", + 39: "CURLE_LDAP_SEARCH_FAILED", + 41: "CURLE_FUNCTION_NOT_FOUND", + 42: "CURLE_ABORTED_BY_CALLBACK", + 43: "CURLE_BAD_FUNCTION_ARGUMENT", + 45: "CURLE_INTERFACE_FAILED", + 47: "CURLE_TOO_MANY_REDIRECTS", + 48: "CURLE_UNKNOWN_OPTION", + 49: "CURLE_TELNET_OPTION_SYNTAX", + 51: "CURLE_PEER_FAILED_VERIFICATION", + 52: "CURLE_GOT_NOTHING", + 53: "CURLE_SSL_ENGINE_NOTFOUND", + 54: "CURLE_SSL_ENGINE_SETFAILED", + 55: "CURLE_SEND_ERROR", + 56: "CURLE_RECV_ERROR", + 58: "CURLE_SSL_CERTPROBLEM", + 59: "CURLE_SSL_CIPHER", + 60: "CURLE_SSL_CACERT", + 61: "CURLE_BAD_CONTENT_ENCODING", + 62: "CURLE_LDAP_INVALID_URL", + 63: "CURLE_FILESIZE_EXCEEDED", + 64: "CURLE_USE_SSL_FAILED", + 65: "CURLE_SEND_FAIL_REWIND", + 66: "CURLE_SSL_ENGINE_INITFAILED", + 67: "CURLE_LOGIN_DENIED", + 68: "CURLE_TFTP_NOTFOUND", + 69: "CURLE_TFTP_PERM", + 70: "CURLE_REMOTE_DISK_FULL", + 71: "CURLE_TFTP_ILLEGAL", + 72: "CURLE_TFTP_UNKNOWNID", + 73: "CURLE_REMOTE_FILE_EXISTS", + 74: "CURLE_TFTP_NOSUCHUSER", + 75: "CURLE_CONV_FAILED", + 76: "CURLE_CONV_REQD", + 77: "CURLE_SSL_CACERT_BADFILE", + 78: "CURLE_REMOTE_FILE_NOT_FOUND", + 79: "CURLE_SSH", + 80: "CURLE_SSL_SHUTDOWN_FAILED", + 81: "CURLE_AGAIN", + 82: "CURLE_SSL_CRL_BADFILE", + 83: "CURLE_SSL_ISSUER_ERROR", + 84: "CURLE_FTP_PRET_FAILED", + 85: "CURLE_RTSP_CSEQ_ERROR", + 86: "CURLE_RTSP_SESSION_ERROR", + 87: "CURLE_FTP_BAD_FILE_LIST", + 88: "CURLE_CHUNK_FAILED", + 89: "CURLE_NO_CONNECTION_AVAILABLE", + 90: "CURLE_SSL_PINNEDPUBKEYNOTMATCH", + 91: "CURLE_SSL_INVALIDCERTSTATUS", + 92: "CURLE_HTTP2_STREAM", + 93: "CURLE_RECURSIVE_API_CALL", + 94: "CURLE_AUTH_ERROR", + 95: "CURLE_HTTP3", + 96: "CURLE_QUIC_CONNECT_ERROR", +} diff --git a/internal/curl/runner.go b/internal/curl/runner.go new file mode 100644 index 0000000..8fb7df1 --- /dev/null +++ b/internal/curl/runner.go @@ -0,0 +1,170 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package curl + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +// Runner handles the execution of a specific curl binary. +// It manages environment setup (e.g., LD_LIBRARY_PATH) required for custom builds. +type Runner struct { + curlPath string + libPath string +} + +// Args defines the execution parameters for a single curl run. +type Args struct { + // Proxy specifies the proxy URL to use. + // Corresponds to the "--proxy" flag. + Proxy string + + // ProxyHeaders specifies custom headers to send to the proxy. + // Each string should be in "Key: Value" format. + // Corresponds to the "--proxy-header" flag. + ProxyHeaders []string + + // ECH specifies the Encrypted ClientHello mode. + // Corresponds to the "--ech" flag. + // Use ECHGrease, ECHTrue, ECHFalse constants. + // If empty (ECHNone), the flag is omitted. + ECH ECHMode + + // Verbose enables verbose output. + // If true, adds "-v". + // If false, adds "-s" (silent mode) by default to keep output clean. + Verbose bool + + // Timeout sets the maximum time allowed for the transfer. + // Corresponds to the "--max-time" flag. + // If 0, no timeout is set. + Timeout time.Duration + + // MeasureStats enables capturing performance metrics using curl's -w flag. + // If true, Stats will be populated in the Result. + MeasureStats bool +} + +// ECHMode defines the available Encrypted ClientHello modes for curl. +type ECHMode string + +const ( + // ECHGrease enables ECH GREASE mode ("--ech grease"). + ECHGrease ECHMode = "grease" + // ECHTrue enables ECH ("--ech true"). + ECHTrue ECHMode = "true" + // ECHFalse disables ECH ("--ech false"). + ECHFalse ECHMode = "false" + // ECHNone indicates that the --ech flag should not be sent. + ECHNone ECHMode = "" +) + +// Result represents the raw outcome of a curl execution. +type Result struct { + // ExitCode is the exit status of the curl process. + // 0 indicates success. See ExitCodeName for error name mapping. + ExitCode int + + // Stdout contains the standard output of the curl command. + Stdout string + + // Stderr contains the standard error of the curl command. + // In verbose mode, this contains debug information and headers. + Stderr string + + // Stats contains performance metrics if MeasureStats was enabled. + Stats Stats +} + +// NewRunner creates a new Runner for the specified curl binary. +// It automatically detects the associated library path (bin/curl -> lib/) +// to ensure shared libraries are found. +func NewRunner(curlPath string) *Runner { + r := &Runner{curlPath: curlPath} + + binDir := filepath.Dir(curlPath) + libDir := filepath.Join(filepath.Dir(binDir), "lib") + if libStat, err := os.Stat(libDir); err == nil && libStat.IsDir() { + r.libPath = libDir + } + + return r +} + +// Run executes curl with the provided arguments and returns the result. +func (r *Runner) Run(url string, args Args) (*Result, error) { + var cmdArgs []string + + if args.Verbose { + cmdArgs = append(cmdArgs, "-v") + } else { + cmdArgs = append(cmdArgs, "-s") + } + + if args.Timeout > 0 { + cmdArgs = append(cmdArgs, "--max-time", strconv.FormatFloat(args.Timeout.Seconds(), 'f', -1, 64)) + } + + if args.Proxy != "" { + cmdArgs = append(cmdArgs, "--proxy", args.Proxy) + } + + for _, h := range args.ProxyHeaders { + cmdArgs = append(cmdArgs, "--proxy-header", h) + } + + if args.ECH != ECHNone { + cmdArgs = append(cmdArgs, "--ech", string(args.ECH)) + } + + if args.MeasureStats { + cmdArgs = append(cmdArgs, "-w", statsFormat) + } + + cmdArgs = append(cmdArgs, url) + cmd := exec.Command(r.curlPath, cmdArgs...) + if r.libPath != "" { + cmd.Env = append(os.Environ(), "LD_LIBRARY_PATH="+r.libPath) + } + + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + result := &Result{} + err := cmd.Run() + result.Stdout = stdout.String() + result.Stderr = stderr.String() + + if args.MeasureStats { + result.Stats = parseStats(result.Stdout) + } + + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + result.ExitCode = exitErr.ExitCode() + } else { + return result, fmt.Errorf("failed to execute curl: %w", err) + } + } + + return result, nil +} diff --git a/internal/curl/stats.go b/internal/curl/stats.go new file mode 100644 index 0000000..a450c55 --- /dev/null +++ b/internal/curl/stats.go @@ -0,0 +1,101 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package curl + +import ( + "strconv" + "strings" + "time" +) + +// Stats captures timing and status metrics from a curl execution. +type Stats struct { + // DNSLookupTimestamp is the cumulative time from the start until the name + // lookup is completed (time_namelookup). + DNSLookupTimestamp time.Duration + + // TCPConnectTimestamp is the cumulative time from the start until the TCP + // connection is completed (time_connect). + TCPConnectTimestamp time.Duration + + // TLSConnectTimestamp is the cumulative time from the start until the + // SSL/TLS handshake is completed (time_appconnect). + TLSConnectTimestamp time.Duration + + // ServerResponseTimestamp is the cumulative time from the start until the + // first byte is received (time_starttransfer). + ServerResponseTimestamp time.Duration + + // TotalTimeTimestamp is the total time from the start until the operation is + // fully completed (time_total). + TotalTimeTimestamp time.Duration + + // HTTPStatus is the HTTP response code (http_code). + HTTPStatus int +} + +const ( + // statsPrefix is the delimiter used to identify the statistics block in the output. + statsPrefix = "\n|||CURL_STATS|||\t" + + // statsFormat is the format string passed to curl's -w flag. + statsFormat = statsPrefix + + "dnslookup:%{time_namelookup}," + + "tcpconnect:%{time_connect}," + + "tlsconnect:%{time_appconnect}," + + "servertime:%{time_starttransfer}," + + "total:%{time_total}," + + "httpstatus:%{http_code}" +) + +// parseStats extracts Stats from the curl output by looking for the statsPrefix. +func parseStats(stdout string) Stats { + var s Stats + idx := strings.LastIndex(stdout, statsPrefix) + if idx == -1 { + return s + } + + raw := strings.TrimSpace(stdout[idx+len(statsPrefix):]) + for part := range strings.SplitSeq(raw, ",") { + kv := strings.Split(part, ":") + if len(kv) != 2 { + continue + } + + key, val := strings.TrimSpace(kv[0]), strings.TrimSpace(kv[1]) + switch key { + case "dnslookup": + s.DNSLookupTimestamp = parseDuration(val) + case "tcpconnect": + s.TCPConnectTimestamp = parseDuration(val) + case "tlsconnect": + s.TLSConnectTimestamp = parseDuration(val) + case "servertime": + s.ServerResponseTimestamp = parseDuration(val) + case "total": + s.TotalTimeTimestamp = parseDuration(val) + case "httpstatus": + s.HTTPStatus, _ = strconv.Atoi(val) + } + } + return s +} + +// parseDuration converts a seconds-based float string to time.Duration. +func parseDuration(s string) time.Duration { + f, _ := strconv.ParseFloat(s, 64) + return time.Duration(f * float64(time.Second)) +} diff --git a/internal/soax/soax.go b/internal/soax/soax.go new file mode 100644 index 0000000..264ff81 --- /dev/null +++ b/internal/soax/soax.go @@ -0,0 +1,118 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package soax + +import ( + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "os" + "strings" +) + +// Config holds the credentials and endpoint configuration for the SOAX service. +type Config struct { + APIKey string `json:"api_key"` + PackageKey string `json:"package_key"` + PackageID string `json:"package_id"` + ProxyHost string `json:"proxy_host"` + ProxyPort int `json:"proxy_port"` +} + +// Client provides methods to interact with the SOAX API and generate proxy configurations. +type Client struct { + cfg *Config +} + +// LoadConfig reads the SOAX configuration from a JSON file. +// If ProxyHost or ProxyPort are missing in the config, default values are used. +func LoadConfig(path string) (*Config, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open config file: %w", err) + } + defer f.Close() + + var cfg Config + if err := json.NewDecoder(f).Decode(&cfg); err != nil { + return nil, fmt.Errorf("failed to decode config json: %w", err) + } + if cfg.ProxyHost == "" { + cfg.ProxyHost = "proxy.soax.com" + } + if cfg.ProxyPort == 0 { + cfg.ProxyPort = 5000 + } + + return &cfg, nil +} + +// NewClient creates a new SOAX Client with the given configuration. +func NewClient(cfg *Config) *Client { + return &Client{cfg: cfg} +} + +// ListISPs retrieves a list of available ISP operators for the specified country code. +// countryISO should be a 2-letter ISO country code (e.g., "US"). +func (c *Client) ListISPs(countryISO string) ([]string, error) { + url := fmt.Sprintf("https://api.soax.com/api/get-country-operators?api_key=%s&package_key=%s&country_iso=%s", + c.cfg.APIKey, c.cfg.PackageKey, strings.ToLower(countryISO)) + + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to fetch ISPs: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API error: status %s, body: %s", resp.Status, string(body)) + } + + var isps []string + if err := json.NewDecoder(resp.Body).Decode(&isps); err != nil { + return nil, fmt.Errorf("failed to decode ISP list: %w", err) + } + return isps, nil +} + +// BuildProxyURL constructs an authenticated HTTPS proxy URL for a specific country and ISP. +// An optional sessionID can be provided for sticky sessions; if empty, a random one is generated. +func (c *Client) BuildProxyURL(countryISO, ispName, sessionID string) string { + if sessionID == "" { + sessionID = generateRandomString(10) + } + + ispName = url.QueryEscape(strings.ToLower(ispName)) + countryISO = strings.ToLower(countryISO) + + proxyUser := fmt.Sprintf("package-%s-country-%s-isp-%s-sessionid-%s-sessionlength-300", + c.cfg.PackageID, countryISO, ispName, sessionID) + + return fmt.Sprintf("https://%s:%s@%s:%d", + proxyUser, c.cfg.PackageKey, c.cfg.ProxyHost, c.cfg.ProxyPort) +} + +func generateRandomString(n int) string { + const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} diff --git a/soaxreport/README.md b/soaxreport/README.md new file mode 100644 index 0000000..ee54c91 --- /dev/null +++ b/soaxreport/README.md @@ -0,0 +1,89 @@ +# SOAX ECH GREASE Report Generation + +This tool tests ECH GREASE compatibility by issuing requests via SOAX proxies. +It iterates through a list of countries and ISPs, running tests with and without +ECH GREASE to simulate diverse network vantage points. + +## Requirements + +You need to build the ECH-enabled `curl` and place it in the workspace directory. See [instructions](../curl/README.md). + +You also need a SOAX configuration file (`soax/cred.json` in the workspace) and a list of ISO country codes. + +### Configuration File Examples + +**SOAX Credentials (`soax/cred.json`)** + +The SOAX configuration file should be a JSON file with the following structure: + +```json +{ + "api_key": "YOUR_API_KEY", + "package_key": "YOUR_PACKAGE_KEY", + "package_id": "YOUR_PACKAGE_ID", + "proxy_host": "proxy.soax.com", + "proxy_port": 5000 +} +``` + +**Country List (`countries.csv`)** + +The countries file should be a CSV file containing country names and their 2-letter ISO codes. Lines starting with `#` are ignored. + +```csv +"United States",US +"United Kingdom",GB +"Germany",DE +# Add more countries as needed +"Virgin Islands, U.S.",VI +``` + +You can download a complete list of country codes from [here](https://raw.githubusercontent.com/datasets/country-list/master/data.csv). + +## Running + +To run the tool, use the `go run` command from the project root directory: + +```sh +go run ./soaxreport --countries workspace/countries.csv --targetDomain www.google.com +``` + +This will: + +1. Load the SOAX credentials (`./workspace/soax/cred.json` by default) and country list. +2. For each country, fetch the list of available ISPs. +3. For each ISP, issue requests to the target domain via a SOAX proxy, once with ECH GREASE and once without. +4. Save the results to `./workspace/soax-results--countries.csv`. + +### Parameters + +* `-workspace `: Directory to store intermediate files. Defaults to `./workspace`. +* `-soax `: Path to SOAX config JSON. Defaults to `./workspace/soax/cred.json`. +* `-countries `: Path to CSV file containing country names and ISO codes (required). +* `-targetDomain `: Target domain to test. Defaults to `www.google.com`. +* `-parallelism `: Maximum number of parallel requests. Defaults to `16`. +* `-verbose`: Enable verbose logging. +* `-maxTime `: Maximum time per curl request. Defaults to `30s`. +* `-curl `: Path to the ECH-enabled curl binary. Defaults to `./workspace/output/bin/curl`. + +### Output Format + +The tool generates a CSV file (`workspace/soax-results--countries.csv`) with the following columns: + +* `domain`: The domain that was tested. +* `country_code`: The 2-letter ISO country code. +* `country_name`: The full name of the country. +* `isp`: The ISP name of the proxy used. +* `asn`: The ASN of the proxy exit node. +* `exit_node_ip`: The IP address of the proxy exit node. +* `exit_node_isp`: The ISP name reported by the proxy exit node (from headers). +* `ech_grease`: `true` if ECH GREASE was enabled for the request, `false` otherwise. +* `error`: Any error that occurred during the request. +* `curl_exit_code`: The exit code returned by the `curl` command. +* `curl_error_name`: The human-readable name corresponding to the `curl` exit code. +* `dns_lookup_ms`: The duration of the DNS lookup. +* `tcp_connection_ms`: The duration of the TCP connection. +* `tls_handshake_ms`: The duration of the TLS handshake. +* `server_time_ms`: The time from the end of the TLS handshake to the first byte of the response. +* `total_time_ms`: The total duration of the request. +* `http_status`: The HTTP status code of the response. diff --git a/soaxreport/main.go b/soaxreport/main.go new file mode 100644 index 0000000..cc08279 --- /dev/null +++ b/soaxreport/main.go @@ -0,0 +1,309 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/csv" + "flag" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/Jigsaw-Code/ech-research/internal/curl" + "github.com/Jigsaw-Code/ech-research/internal/soax" + "github.com/Jigsaw-Code/ech-research/internal/workspace" + "golang.org/x/sync/semaphore" +) + +type TestResult struct { + Domain string + Country string + CountryName string + ISP string + ASN string + ExitNodeIP string + ExitNodeISP string + ECHGrease bool + Error string + CurlExitCode int + CurlErrorName string + DNSLookup time.Duration + TCPConnection time.Duration + TLSHandshake time.Duration + ServerTime time.Duration + TotalTime time.Duration + HTTPStatus int +} + +func runSoaxTest( + runner *curl.Runner, + domain string, + country string, + countryName string, + isp string, + proxyURL string, + echGrease bool, + maxTime time.Duration, +) TestResult { + result := TestResult{ + Domain: domain, + Country: country, + CountryName: countryName, + ISP: isp, + ECHGrease: echGrease, + } + + echMode := curl.ECHFalse + if echGrease { + echMode = curl.ECHGrease + } + + url := "https://" + domain + res, err := runner.Run(url, curl.Args{ + Proxy: proxyURL, + ProxyHeaders: []string{"Respond-With: ip,isp,asn"}, + ECH: echMode, + Timeout: maxTime, + Verbose: true, // Required to capture response headers + MeasureStats: true, + }) + + result.CurlExitCode = res.ExitCode + result.CurlErrorName = curl.ExitCodeName(res.ExitCode) + result.HTTPStatus = res.Stats.HTTPStatus + result.DNSLookup = res.Stats.DNSLookupTimestamp + result.TCPConnection = res.Stats.TCPConnectTimestamp + result.TLSHandshake = res.Stats.TLSConnectTimestamp + result.ServerTime = res.Stats.ServerResponseTimestamp + result.TotalTime = res.Stats.TotalTimeTimestamp + + if err != nil { + result.Error = err.Error() + } + + // Parse metadata from Stderr (SOAX specific headers in CONNECT response) + for line := range strings.SplitSeq(res.Stderr, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "< Node-") { + continue + } + kv := strings.SplitN(strings.TrimPrefix(line, "< Node-"), ":", 2) + if len(kv) != 2 { + continue + } + key := strings.ToLower(strings.TrimSpace(kv[0])) + val := strings.TrimSpace(kv[1]) + switch key { + case "asn": + result.ASN = val + case "ip": + result.ExitNodeIP = val + case "isp": + result.ExitNodeISP = val + } + } + + return result +} + +type Country struct { + Name string + Code string +} + +func loadCountries(path string) ([]Country, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + var countries []Country + reader := csv.NewReader(f) + reader.Comment = '#' // Support skipping lines starting with # + reader.FieldsPerRecord = 2 + + records, err := reader.ReadAll() + if err != nil { + return nil, fmt.Errorf("failed to read countries CSV: %w", err) + } + + for i, record := range records { + name := strings.TrimSpace(record[0]) + code := strings.TrimSpace(record[1]) + + // Skip header row if present + if i == 0 && strings.EqualFold(name, "Name") && strings.EqualFold(code, "Code") { + continue + } + + countries = append(countries, Country{ + Name: name, + Code: code, + }) + } + return countries, nil +} + +func main() { + var ( + workspaceFlag = flag.String("workspace", "./workspace", "Directory to store intermediate files") + soaxConfigFlag = flag.String("soax", "", "Path to SOAX config JSON") + countriesFlag = flag.String("countries", "", "Path to file containing ISO country codes") + targetDomainFlag = flag.String("targetDomain", "www.google.com", "Target domain to test") + verboseFlag = flag.Bool("verbose", false, "Enable verbose logging") + maxTimeFlag = flag.Duration("maxTime", 30*time.Second, "Maximum time per curl request") + curlPathFlag = flag.String("curl", "", "Path to the ECH-enabled curl binary") + parallelismFlag = flag.Int("parallelism", 16, "Maximum number of parallel requests") + ) + flag.Parse() + + if *verboseFlag { + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))) + } else { + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))) + } + + // Set up workspace directory + workspaceDir := workspace.EnsureWorkspace(*workspaceFlag) + + // Determine curl binary path + curlPath := *curlPathFlag + if curlPath == "" { + curlPath = filepath.Join(workspaceDir, "output", "bin", "curl") + } + runner := curl.NewRunner(curlPath) + + // Load SOAX config + soaxConfigPath := *soaxConfigFlag + if soaxConfigPath == "" { + soaxConfigPath = filepath.Join(workspaceDir, "soax", "cred.json") + } + cfg, err := soax.LoadConfig(soaxConfigPath) + if err != nil { + slog.Error("Failed to load SOAX config", "path", soaxConfigPath, "error", err) + os.Exit(1) + } + client := soax.NewClient(cfg) + + // Load countries + if *countriesFlag == "" { + slog.Error("The --countries flag is required") + os.Exit(1) + } + countries, err := loadCountries(*countriesFlag) + if err != nil { + slog.Error("Failed to load countries list", "path", *countriesFlag, "error", err) + os.Exit(1) + } + + // Create output CSV file + sanitizedDomain := strings.ReplaceAll(*targetDomainFlag, ".", "_") + outputFilename := filepath.Join(workspaceDir, fmt.Sprintf("soax-results-%s-countries%d.csv", sanitizedDomain, len(countries))) + outputFile, err := os.Create(outputFilename) + if err != nil { + slog.Error("Failed to create output CSV file", "path", outputFilename, "error", err) + os.Exit(1) + } + defer outputFile.Close() + + resultsCh := make(chan TestResult, 2*len(countries)*(*parallelismFlag)) + + var csvWg sync.WaitGroup + csvWg.Add(1) + go func() { + defer csvWg.Done() + csvWriter := csv.NewWriter(outputFile) + defer csvWriter.Flush() + + header := []string{ + "domain", "country_code", "country_name", "isp", "asn", "exit_node_ip", "exit_node_isp", "ech_grease", "error", + "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", + "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", + } + if err := csvWriter.Write(header); err != nil { + slog.Error("Failed to write CSV header", "error", err) + } + + for r := range resultsCh { + record := []string{ + r.Domain, r.Country, r.CountryName, r.ISP, r.ASN, r.ExitNodeIP, r.ExitNodeISP, strconv.FormatBool(r.ECHGrease), r.Error, + strconv.Itoa(r.CurlExitCode), r.CurlErrorName, + strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), + strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), + strconv.FormatInt(r.TLSHandshake.Milliseconds(), 10), + strconv.FormatInt(r.ServerTime.Milliseconds(), 10), + strconv.FormatInt(r.TotalTime.Milliseconds(), 10), + strconv.Itoa(r.HTTPStatus), + } + if err := csvWriter.Write(record); err != nil { + slog.Error("Failed to write record to CSV", "error", err) + } + } + }() + + domain := *targetDomainFlag + runSessionID := time.Now().Format("0102150405") + sem := semaphore.NewWeighted(int64(*parallelismFlag)) + var wg sync.WaitGroup + var total, finished atomic.Int32 + + for _, country := range countries { + slog.Debug("Processing country", "name", country.Name, "code", country.Code) + + isps, err := client.ListISPs(country.Code) + if err != nil { + slog.Error("Failed to fetch ISPs", "country", country.Code, "error", err) + continue + } + + total.Add(int32(len(isps) * 2)) + for i, isp := range isps { + wg.Add(2) + sessionID := fmt.Sprintf("%s%s%d", runSessionID, country.Code, i) + + startTest := func(c Country, isp, sid string, ech bool) { + defer wg.Done() + if err := sem.Acquire(context.Background(), 1); err != nil { + slog.Error("Failed to acquire semaphore", "error", err) + return + } + defer sem.Release(1) + + proxyURL := client.BuildProxyURL(c.Code, isp, sid) + slog.Debug("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", ech, "session", sid) + resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, ech, *maxTimeFlag) + progress := fmt.Sprintf("%d/%d", finished.Add(1), total.Load()) + slog.Info("Finished", "country", c.Code, "isp", isp, "progress", progress) + } + + go startTest(country, isp, sessionID, false) + go startTest(country, isp, sessionID, true) + } + } + + wg.Wait() + close(resultsCh) + csvWg.Wait() + + slog.Info("Done. Results saved to", "path", outputFilename) +}