From 086aa5fb2e79589e39cdefae98cdfe19ad3f1723 Mon Sep 17 00:00:00 2001 From: Mani Chandrasekar Date: Wed, 11 Jun 2025 16:46:21 -0700 Subject: [PATCH 1/5] Support Glue REST queries with Iceberg-Go CLI --- cmd/iceberg/main.go | 24 ++++++++++++++++++ config/config.go | 19 ++++++++++---- config/config_test.go | 58 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 5 deletions(-) diff --git a/cmd/iceberg/main.go b/cmd/iceberg/main.go index 74605792c..6f8b66b64 100644 --- a/cmd/iceberg/main.go +++ b/cmd/iceberg/main.go @@ -19,9 +19,11 @@ package main import ( "context" + "crypto/tls" "errors" "fmt" "log" + "net/url" "os" "strings" @@ -160,6 +162,28 @@ func main() { if len(cfg.Cred) > 0 { opts = append(opts, rest.WithCredential(cfg.Cred)) } + if fileCfg != nil && &fileCfg.RestConfig != nil { + restCfg := fileCfg.RestConfig + if restCfg.SigV4Enabled { + opts = append(opts, rest.WithSigV4()) + } + + if len(restCfg.SigV4Region) > 0 && len(restCfg.SigV4Service) > 0 { + opts = append(opts, rest.WithSigV4RegionSvc(restCfg.SigV4Region, restCfg.SigV4Service)) + } + + if len(restCfg.AuthUrl) > 0 { + authUri, err := url.Parse(restCfg.AuthUrl) + if err != nil { + log.Fatal(err) + } + opts = append(opts, rest.WithAuthURI(authUri)) + } + + if restCfg.TlsSkipVerify { + opts = append(opts, rest.WithTLSConfig(&tls.Config{InsecureSkipVerify: true})) + } + } if len(cfg.Warehouse) > 0 { opts = append(opts, rest.WithWarehouseLocation(cfg.Warehouse)) diff --git a/config/config.go b/config/config.go index 3d645ddee..2808ff9ed 100644 --- a/config/config.go +++ b/config/config.go @@ -36,11 +36,20 @@ type Config struct { } type CatalogConfig struct { - CatalogType string `yaml:"type"` - URI string `yaml:"uri"` - Output string `yaml:"output"` - Credential string `yaml:"credential"` - Warehouse string `yaml:"warehouse"` + CatalogType string `yaml:"type"` + URI string `yaml:"uri"` + Output string `yaml:"output"` + Credential string `yaml:"credential"` + Warehouse string `yaml:"warehouse"` + RestConfig RestCatalogConfig `yaml:"rest-config"` +} + +type RestCatalogConfig struct { + AuthUrl string `yaml:"auth-url"` + SigV4Enabled bool `yaml:"sigv4-enabled"` + SigV4Region string `yaml:"sigv4-region"` + SigV4Service string `yaml:"sigv4-service"` + TlsSkipVerify bool `yaml:"tls-skip-verify"` } func LoadConfig(configPath string) []byte { diff --git a/config/config_test.go b/config/config_test.go index c176b6382..42726b523 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -78,6 +78,38 @@ catalog: Warehouse: "catalog_name", }, }, + // catalog with rest-config + { + []byte(` +catalog: + rest-catalog: + type: rest + uri: https://glue.us-east-1.amazonaws.com/iceberg + output: json + credential: client-id:client-secret + warehouse: 123456789012 + rest-config: + auth-url: https://auth.example.com + sigv4-enabled: true + sigv4-region: us-east-1 + sigv4-service: glue + tls-skip-verify: false +`), "rest-catalog", + &CatalogConfig{ + CatalogType: "rest", + URI: "https://glue.us-east-1.amazonaws.com/iceberg", + Output: "json", + Credential: "client-id:client-secret", + Warehouse: "123456789012", + RestConfig: RestCatalogConfig{ + AuthUrl: "https://auth.example.com", + SigV4Enabled: true, + SigV4Region: "us-east-1", + SigV4Service: "glue", + TlsSkipVerify: false, + }, + }, + }, } func TestParseConfig(t *testing.T) { @@ -87,3 +119,29 @@ func TestParseConfig(t *testing.T) { assert.Equal(t, tt.expected, actual) } } + +func TestRestCatalogConfig(t *testing.T) { + yamlData := []byte(` +catalog: + default: + type: rest + uri: https://glue.us-east-1.amazonaws.com/iceberg + output: json + rest-config: + auth-url: https://auth.example.com + sigv4-enabled: true + sigv4-region: us-east-1 + sigv4-service: glue + tls-skip-verify: false +`) + + config := ParseConfig(yamlData, "default") + assert.NotNil(t, config) + + // Test RestCatalogConfig fields + assert.Equal(t, "https://auth.example.com", config.RestConfig.AuthUrl) + assert.True(t, config.RestConfig.SigV4Enabled) + assert.Equal(t, "us-east-1", config.RestConfig.SigV4Region) + assert.Equal(t, "glue", config.RestConfig.SigV4Service) + assert.False(t, config.RestConfig.TlsSkipVerify) +} From 6d90806d71ac353d6f044c7193d7505b7141e141 Mon Sep 17 00:00:00 2001 From: Mani Chandrasekar Date: Thu, 12 Jun 2025 09:49:09 -0700 Subject: [PATCH 2/5] Fix build failure due to lint check --- cmd/iceberg/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/iceberg/main.go b/cmd/iceberg/main.go index 6f8b66b64..b0faa7a11 100644 --- a/cmd/iceberg/main.go +++ b/cmd/iceberg/main.go @@ -162,7 +162,7 @@ func main() { if len(cfg.Cred) > 0 { opts = append(opts, rest.WithCredential(cfg.Cred)) } - if fileCfg != nil && &fileCfg.RestConfig != nil { + if fileCfg != nil { restCfg := fileCfg.RestConfig if restCfg.SigV4Enabled { opts = append(opts, rest.WithSigV4()) From 38bc5bd21d2230e0532ecfed72630df1e4f46e28 Mon Sep 17 00:00:00 2001 From: Mani Chandrasekar Date: Sun, 6 Jul 2025 19:56:34 -0700 Subject: [PATCH 3/5] Use rest-config property from command line configuration and config file --- cmd/iceberg/main.go | 25 ++++++++++++++++--------- config/config.go | 28 ++++++++++++++++++++++------ config/config_test.go | 36 +++++++++++------------------------- 3 files changed, 49 insertions(+), 40 deletions(-) diff --git a/cmd/iceberg/main.go b/cmd/iceberg/main.go index b0faa7a11..9955d8a9f 100644 --- a/cmd/iceberg/main.go +++ b/cmd/iceberg/main.go @@ -84,7 +84,9 @@ Options: --description TEXT specify a description for the namespace --location-uri TEXT specify a location URI for the namespace --schema JSON specify table schema in json (for create table use only) - Ex: [{"name":"id","type":"int","required":false,"doc":"unique id"}]` + Ex: [{"name":"id","type":"int","required":false,"doc":"unique id"}] + --rest-config TEST specify the REST configuration to use + Ex: sigv4-enabled=true,sigv4-region=us-east-1,sigv4-service=glue` type Config struct { List bool `docopt:"list"` @@ -125,6 +127,7 @@ type Config struct { Description string `docopt:"--description"` LocationURI string `docopt:"--location-uri"` SchemaStr string `docopt:"--schema"` + RestConfig string `docopt:"--rest-config"` } func main() { @@ -162,25 +165,26 @@ func main() { if len(cfg.Cred) > 0 { opts = append(opts, rest.WithCredential(cfg.Cred)) } - if fileCfg != nil { - restCfg := fileCfg.RestConfig - if restCfg.SigV4Enabled { + if len(cfg.RestConfig) > 0 { + restCfg := config.ParseRestConfig(cfg.RestConfig) + + if strings.ToLower(restCfg["sigv4-enabled"]) == "true" { opts = append(opts, rest.WithSigV4()) } - if len(restCfg.SigV4Region) > 0 && len(restCfg.SigV4Service) > 0 { - opts = append(opts, rest.WithSigV4RegionSvc(restCfg.SigV4Region, restCfg.SigV4Service)) + if len(restCfg["sigv4-region"]) > 0 && len(restCfg["sigv4-service"]) > 0 { + opts = append(opts, rest.WithSigV4RegionSvc(restCfg["sigv4-region"], restCfg["sigv4-service"])) } - if len(restCfg.AuthUrl) > 0 { - authUri, err := url.Parse(restCfg.AuthUrl) + if len(restCfg["auth-url"]) > 0 { + authUri, err := url.Parse(restCfg["auth-url"]) if err != nil { log.Fatal(err) } opts = append(opts, rest.WithAuthURI(authUri)) } - if restCfg.TlsSkipVerify { + if restCfg["tls-skip-verify"] == "true" { opts = append(opts, rest.WithTLSConfig(&tls.Config{InsecureSkipVerify: true})) } } @@ -493,4 +497,7 @@ func mergeConf(fileConf *config.CatalogConfig, resConfig *Config) { if len(resConfig.Warehouse) == 0 { resConfig.Warehouse = fileConf.Warehouse } + if len(resConfig.RestConfig) == 0 { + resConfig.RestConfig = fileConf.RestConfig + } } diff --git a/config/config.go b/config/config.go index 2808ff9ed..3c2e3c342 100644 --- a/config/config.go +++ b/config/config.go @@ -20,6 +20,7 @@ package config import ( "os" "path/filepath" + "strings" "gopkg.in/yaml.v3" ) @@ -36,12 +37,12 @@ type Config struct { } type CatalogConfig struct { - CatalogType string `yaml:"type"` - URI string `yaml:"uri"` - Output string `yaml:"output"` - Credential string `yaml:"credential"` - Warehouse string `yaml:"warehouse"` - RestConfig RestCatalogConfig `yaml:"rest-config"` + CatalogType string `yaml:"type"` + URI string `yaml:"uri"` + Output string `yaml:"output"` + Credential string `yaml:"credential"` + Warehouse string `yaml:"warehouse"` + RestConfig string `yaml:"rest-config"` } type RestCatalogConfig struct { @@ -85,6 +86,21 @@ func ParseConfig(file []byte, catalogName string) *CatalogConfig { return &res } +func ParseRestConfig(restConfig string) map[string]string { + result := make(map[string]string) + if restConfig == "" { + return result + } + + pairs := strings.Split(restConfig, ",") + for _, pair := range pairs { + if kv := strings.SplitN(pair, "=", 2); len(kv) == 2 { + result[kv[0]] = kv[1] + } + } + return result +} + func fromConfigFiles() Config { dir := os.Getenv("GOICEBERG_HOME") if dir != "" { diff --git a/config/config_test.go b/config/config_test.go index 42726b523..ebd6fd51e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -88,12 +88,7 @@ catalog: output: json credential: client-id:client-secret warehouse: 123456789012 - rest-config: - auth-url: https://auth.example.com - sigv4-enabled: true - sigv4-region: us-east-1 - sigv4-service: glue - tls-skip-verify: false + rest-config: auth-url=https://auth.example.com,sigv4-enabled=true,sigv4-region=us-east-1,sigv4-service=glue,tls-skip-verify=false `), "rest-catalog", &CatalogConfig{ CatalogType: "rest", @@ -101,13 +96,7 @@ catalog: Output: "json", Credential: "client-id:client-secret", Warehouse: "123456789012", - RestConfig: RestCatalogConfig{ - AuthUrl: "https://auth.example.com", - SigV4Enabled: true, - SigV4Region: "us-east-1", - SigV4Service: "glue", - TlsSkipVerify: false, - }, + RestConfig: "auth-url=https://auth.example.com,sigv4-enabled=true,sigv4-region=us-east-1,sigv4-service=glue,tls-skip-verify=false", }, }, } @@ -127,21 +116,18 @@ catalog: type: rest uri: https://glue.us-east-1.amazonaws.com/iceberg output: json - rest-config: - auth-url: https://auth.example.com - sigv4-enabled: true - sigv4-region: us-east-1 - sigv4-service: glue - tls-skip-verify: false + rest-config: auth-url=https://auth.example.com,sigv4-enabled=true,sigv4-region=us-east-1,sigv4-service=glue,tls-skip-verify=false `) config := ParseConfig(yamlData, "default") assert.NotNil(t, config) + restConfig := ParseRestConfig(config.RestConfig) + assert.NotEmpty(t, restConfig) - // Test RestCatalogConfig fields - assert.Equal(t, "https://auth.example.com", config.RestConfig.AuthUrl) - assert.True(t, config.RestConfig.SigV4Enabled) - assert.Equal(t, "us-east-1", config.RestConfig.SigV4Region) - assert.Equal(t, "glue", config.RestConfig.SigV4Service) - assert.False(t, config.RestConfig.TlsSkipVerify) + // Test RestCatalog config fields + assert.Equal(t, "https://auth.example.com", restConfig["auth-url"]) + assert.Equal(t, "true", restConfig["sigv4-enabled"]) + assert.Equal(t, "us-east-1", restConfig["sigv4-region"]) + assert.Equal(t, "glue", restConfig["sigv4-service"]) + assert.Equal(t, "false", restConfig["tls-skip-verify"]) } From f63960419e89a0b6a3047673d7128bd369346d46 Mon Sep 17 00:00:00 2001 From: Mani Chandrasekar Date: Sun, 6 Jul 2025 20:00:54 -0700 Subject: [PATCH 4/5] fix typo --- cmd/iceberg/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/iceberg/main.go b/cmd/iceberg/main.go index 1ebb66dab..ffa747e95 100644 --- a/cmd/iceberg/main.go +++ b/cmd/iceberg/main.go @@ -86,7 +86,7 @@ Options: --location-uri TEXT specify a location URI for the namespace --schema JSON specify table schema in json (for create table use only) Ex: [{"name":"id","type":"int","required":false,"doc":"unique id"}] - --rest-config TEST specify the REST configuration to use + --rest-config TEXT specify the REST configuration to use Ex: sigv4-enabled=true,sigv4-region=us-east-1,sigv4-service=glue` type Config struct { From 6d6eb6a4212b27da0d668e2ee856bf455b31eedb Mon Sep 17 00:00:00 2001 From: Mani Chandrasekar Date: Sun, 6 Jul 2025 20:04:25 -0700 Subject: [PATCH 5/5] fix indentation and removed unused code --- cmd/iceberg/main.go | 2 +- config/config.go | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/cmd/iceberg/main.go b/cmd/iceberg/main.go index ffa747e95..7ebe8a403 100644 --- a/cmd/iceberg/main.go +++ b/cmd/iceberg/main.go @@ -87,7 +87,7 @@ Options: --schema JSON specify table schema in json (for create table use only) Ex: [{"name":"id","type":"int","required":false,"doc":"unique id"}] --rest-config TEXT specify the REST configuration to use - Ex: sigv4-enabled=true,sigv4-region=us-east-1,sigv4-service=glue` + Ex: sigv4-enabled=true,sigv4-region=us-east-1,sigv4-service=glue` type Config struct { List bool `docopt:"list"` diff --git a/config/config.go b/config/config.go index 3c2e3c342..02a45df74 100644 --- a/config/config.go +++ b/config/config.go @@ -45,14 +45,6 @@ type CatalogConfig struct { RestConfig string `yaml:"rest-config"` } -type RestCatalogConfig struct { - AuthUrl string `yaml:"auth-url"` - SigV4Enabled bool `yaml:"sigv4-enabled"` - SigV4Region string `yaml:"sigv4-region"` - SigV4Service string `yaml:"sigv4-service"` - TlsSkipVerify bool `yaml:"tls-skip-verify"` -} - func LoadConfig(configPath string) []byte { var path string if len(configPath) > 0 {