From 714babe7893dc872df9e9eaa678d96d6e3c22649 Mon Sep 17 00:00:00 2001 From: Jan-Robin Aumann Date: Wed, 17 Sep 2025 12:32:11 +0200 Subject: [PATCH 1/4] add support for query params in db uri in cf deployments --- src/jetstream/datastore/database_cf_config.go | 18 ++++++++++----- src/jetstream/datastore/datastore.go | 23 ++++++++++--------- src/jetstream/datastore/datastore_test.go | 21 ++++++++++++++++- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/src/jetstream/datastore/database_cf_config.go b/src/jetstream/datastore/database_cf_config.go index 107226e13f..76812d2acc 100644 --- a/src/jetstream/datastore/database_cf_config.go +++ b/src/jetstream/datastore/database_cf_config.go @@ -135,7 +135,7 @@ func findDatabaseConfig(vcapServices map[string][]VCAPService, db *DatabaseConfi return false } - db.Username, db.Password, db.Host, db.Port, db.Database, err = findDatabaseConfigurationFromURI(uri, defaultDBProviderPort(service)) + db.Username, db.Password, db.Host, db.Port, db.Database, db.QueryParams, err = findDatabaseConfigurationFromURI(uri, defaultDBProviderPort(service)) if err != nil { log.Warnf("Failed to find Cloud Foundry service config from `%v` (failed to parse)", DB_URI) @@ -197,12 +197,12 @@ func stringInSlice(a string, list []string) bool { return false } -func findDatabaseConfigurationFromURI(uri string, defaultPort int) (string, string, string, int, string, error) { - re := regexp.MustCompile(`(?P.+)://(?P[^:]+)(?::(?P.+))?@(?P[^:]+)(?::(?P.+))?\/(?P.+)`) +func findDatabaseConfigurationFromURI(uri string, defaultPort int) (string, string, string, int, string, map[string]string, error) { + re := regexp.MustCompile(`(?P.+)://(?P[^:]+)(?::(?P.+))?@(?P[^:]+)(?::(?P.+))?\/(?P[^?]+)(?:\?(?P.*))`) n1 := re.SubexpNames() matches := re.FindAllStringSubmatch(uri, -1) if len(matches) < 1 { - return "", "", "", 0, "", errors.New("failed to parse database URI") + return "", "", "", 0, "", map[string]string{}, errors.New("failed to parse database URI") } r2 := matches[0] @@ -222,9 +222,15 @@ func findDatabaseConfigurationFromURI(uri string, defaultPort int) (string, stri port = defaultPort } dbname := md["dbname"] + queryparamsraw := md["queryparams"] + queryparams := make(map[string]string) + for _, keyvalue := range strings.Split(queryparamsraw, "&") { + if key, value, valid := strings.Cut(keyvalue, "="); valid { + queryparams[key] = value + } + } - return username, password, host, port, dbname, nil - + return username, password, host, port, dbname, queryparams, nil } func defaultDBProviderPort(service VCAPService) int { diff --git a/src/jetstream/datastore/datastore.go b/src/jetstream/datastore/datastore.go index 618450ba47..3afbc0d115 100644 --- a/src/jetstream/datastore/datastore.go +++ b/src/jetstream/datastore/datastore.go @@ -77,17 +77,18 @@ func GetColumnNames(databaseName string, exclude ...string) []string { // DatabaseConfig represents the connection configuration parameters type DatabaseConfig struct { - DatabaseProvider string `configName:"DATABASE_PROVIDER"` - Username string `configName:"DB_USER"` - Password string `configName:"DB_PASSWORD"` - Database string `configName:"DB_DATABASE_NAME"` - Host string `configName:"DB_HOST"` - Port int `configName:"DB_PORT"` - SSLMode string `configName:"DB_SSL_MODE"` - ConnectionTimeoutInSecs int `configName:"DB_CONNECT_TIMEOUT_IN_SECS"` - SSLCertificate string `configName:"DB_CERT"` - SSLKey string `configName:"DB_CERT_KEY"` - SSLRootCertificate string `configName:"DB_ROOT_CERT"` + DatabaseProvider string `configName:"DATABASE_PROVIDER"` + Username string `configName:"DB_USER"` + Password string `configName:"DB_PASSWORD"` + Database string `configName:"DB_DATABASE_NAME"` + Host string `configName:"DB_HOST"` + Port int `configName:"DB_PORT"` + SSLMode string `configName:"DB_SSL_MODE"` + ConnectionTimeoutInSecs int `configName:"DB_CONNECT_TIMEOUT_IN_SECS"` + SSLCertificate string `configName:"DB_CERT"` + SSLKey string `configName:"DB_CERT_KEY"` + SSLRootCertificate string `configName:"DB_ROOT_CERT"` + QueryParams map[string]string `configName:"DB_QUERY_PARAMS"` } // SSLValidationMode is the PostgreSQL driver SSL validation modes diff --git a/src/jetstream/datastore/datastore_test.go b/src/jetstream/datastore/datastore_test.go index d4157bd3ee..24e945d72c 100644 --- a/src/jetstream/datastore/datastore_test.go +++ b/src/jetstream/datastore/datastore_test.go @@ -429,7 +429,7 @@ func TestDatastore(t *testing.T) { mockEnvVarsMap := make(map[string]string) mockEnvVarsMap["DB_SSL_MODE"] = mockSSLModeVerifyCA - mockEnvVarsMap["VCAP_SERVICES"] = `{"cf-postgresql-service": [ { "name": "mock-stratos-ssl", "credentials": { "cacrt": "mockcert", "host": "mockhost", "name": "mockname", "password": "mockpassword", "port": 5432, "username": "mockusername", "uri": "postgres://mockuser:mockpassword@mockhost:5432/mockname" } } ] }` + mockEnvVarsMap["VCAP_SERVICES"] = `{"cf-postgresql-service": [ { "name": "mock-stratos-ssl", "credentials": { "cacrt": "mockcert", "host": "mockhost", "name": "mockname", "password": "mockpassword", "port": 5432, "username": "mockusername", "uri": "postgres://mockusername:mockpassword@mockhost:5432/mockname?mockquery=true" } } ] }` mockVarSet := env.NewVarSet(env.WithMapLookup(mockEnvVarsMap)) @@ -453,5 +453,24 @@ func TestDatastore(t *testing.T) { So(mockDatabaseConfigSSL.SSLMode, ShouldEqual, mockSSLModeVerifyCA) }) }) + + Convey("when the cloudfoundry database config is present but incomplete", func() { + + Convey("err will be nil and fallback to uri will be used", func() { + mockEnvVarsMap["VCAP_SERVICES"] = `{"cf-postgresql-service": [ { "name": "mock-stratos-ssl", "credentials": { "cacrt": "mockcert", "host": "mockhost", "password": "mockpassword", "port": 5432, "username": "mockusername", "uri": "postgres://mockusername:mockpassword@mockhost:5432/mockname?mockquery=true" } } ] }` + mockVarSet := env.NewVarSet(env.WithMapLookup(mockEnvVarsMap)) + + _, err := ParseCFEnvs(&mockDatabaseConfigSSL, mockVarSet) + So(err, ShouldBeNil) + So(mockDatabaseConfigSSL.SSLRootCertificate, ShouldContainSubstring, "postgres-ssl-") + So(mockDatabaseConfigSSL.SSLRootCertificate, ShouldEndWith, ".crt") + So(mockDatabaseConfigSSL.Username, ShouldEqual, "mockusername") + So(mockDatabaseConfigSSL.Password, ShouldEqual, "mockpassword") + So(mockDatabaseConfigSSL.Database, ShouldEqual, "mockname") + So(mockDatabaseConfigSSL.Host, ShouldEqual, "mockhost") + So(mockDatabaseConfigSSL.SSLMode, ShouldEqual, mockSSLModeVerifyCA) + So(mockDatabaseConfigSSL.QueryParams, ShouldEqual, map[string]string{"mockquery": "true"}) + }) + }) }) } From 15118d4b46cac47f180eb436d977c02b48406b64 Mon Sep 17 00:00:00 2001 From: Jan-Robin Aumann Date: Wed, 17 Sep 2025 12:32:31 +0200 Subject: [PATCH 2/4] debug log for cf deployments credentials --- src/jetstream/datastore/database_cf_config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jetstream/datastore/database_cf_config.go b/src/jetstream/datastore/database_cf_config.go index 76812d2acc..83344b901a 100644 --- a/src/jetstream/datastore/database_cf_config.go +++ b/src/jetstream/datastore/database_cf_config.go @@ -86,6 +86,7 @@ func findDatabaseConfig(vcapServices map[string][]VCAPService, db *DatabaseConfi db.Port, _ = strconv.Atoi(getDBCredentialsValue(dbCredentials["port"])) // Note - Both isPostgresService and isMySQLService look at the credentials uri & tags if isPostgresService(service) { + log.Debugf("REMOVE ME: dbCredentials %+v", dbCredentials) db.DatabaseProvider = "pgsql" db.Database = getDBCredentialsValue(dbCredentials["name"]) if db.Database == "" { // If database name is empty, use dbname From cf85dea6d231807e70413f08430414133bd84ae0 Mon Sep 17 00:00:00 2001 From: Jan-Robin Aumann Date: Wed, 17 Sep 2025 12:51:23 +0200 Subject: [PATCH 3/4] Revert "debug log for cf deployments credentials" This reverts commit 074f5903a273a22f3ce431eb3830705c49cf4101. --- src/jetstream/datastore/database_cf_config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/jetstream/datastore/database_cf_config.go b/src/jetstream/datastore/database_cf_config.go index 83344b901a..76812d2acc 100644 --- a/src/jetstream/datastore/database_cf_config.go +++ b/src/jetstream/datastore/database_cf_config.go @@ -86,7 +86,6 @@ func findDatabaseConfig(vcapServices map[string][]VCAPService, db *DatabaseConfi db.Port, _ = strconv.Atoi(getDBCredentialsValue(dbCredentials["port"])) // Note - Both isPostgresService and isMySQLService look at the credentials uri & tags if isPostgresService(service) { - log.Debugf("REMOVE ME: dbCredentials %+v", dbCredentials) db.DatabaseProvider = "pgsql" db.Database = getDBCredentialsValue(dbCredentials["name"]) if db.Database == "" { // If database name is empty, use dbname From 01c629d7bc40dbe5e8bbbd41bbd35a79f3e41441 Mon Sep 17 00:00:00 2001 From: Jan-Robin Aumann Date: Tue, 23 Sep 2025 09:37:11 +0200 Subject: [PATCH 4/4] address feedaback from PR: fix regex for uri to make query params truly optional and cover with tests --- src/jetstream/datastore/database_cf_config.go | 2 +- src/jetstream/datastore/datastore_test.go | 42 +++++++++++++------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/jetstream/datastore/database_cf_config.go b/src/jetstream/datastore/database_cf_config.go index 76812d2acc..3abaf90cb2 100644 --- a/src/jetstream/datastore/database_cf_config.go +++ b/src/jetstream/datastore/database_cf_config.go @@ -198,7 +198,7 @@ func stringInSlice(a string, list []string) bool { } func findDatabaseConfigurationFromURI(uri string, defaultPort int) (string, string, string, int, string, map[string]string, error) { - re := regexp.MustCompile(`(?P.+)://(?P[^:]+)(?::(?P.+))?@(?P[^:]+)(?::(?P.+))?\/(?P[^?]+)(?:\?(?P.*))`) + re := regexp.MustCompile(`(?P.+)://(?P[^:]+)(?::(?P.+))?@(?P[^:]+)(?::(?P.+))?\/(?P[^?]+)(?:\?(?P.*))*`) n1 := re.SubexpNames() matches := re.FindAllStringSubmatch(uri, -1) if len(matches) < 1 { diff --git a/src/jetstream/datastore/datastore_test.go b/src/jetstream/datastore/datastore_test.go index 24e945d72c..1ac11bc13e 100644 --- a/src/jetstream/datastore/datastore_test.go +++ b/src/jetstream/datastore/datastore_test.go @@ -457,19 +457,37 @@ func TestDatastore(t *testing.T) { Convey("when the cloudfoundry database config is present but incomplete", func() { Convey("err will be nil and fallback to uri will be used", func() { - mockEnvVarsMap["VCAP_SERVICES"] = `{"cf-postgresql-service": [ { "name": "mock-stratos-ssl", "credentials": { "cacrt": "mockcert", "host": "mockhost", "password": "mockpassword", "port": 5432, "username": "mockusername", "uri": "postgres://mockusername:mockpassword@mockhost:5432/mockname?mockquery=true" } } ] }` - mockVarSet := env.NewVarSet(env.WithMapLookup(mockEnvVarsMap)) + Convey("with query params present", func() { + mockEnvVarsMap["VCAP_SERVICES"] = `{"cf-postgresql-service": [ { "name": "mock-stratos-ssl", "credentials": { "cacrt": "mockcert", "host": "mockhost", "password": "mockpassword", "port": 5432, "username": "mockusername", "uri": "postgres://mockusername:mockpassword@mockhost:5432/mockname?mockquery=true" } } ] }` + mockVarSet := env.NewVarSet(env.WithMapLookup(mockEnvVarsMap)) - _, err := ParseCFEnvs(&mockDatabaseConfigSSL, mockVarSet) - So(err, ShouldBeNil) - So(mockDatabaseConfigSSL.SSLRootCertificate, ShouldContainSubstring, "postgres-ssl-") - So(mockDatabaseConfigSSL.SSLRootCertificate, ShouldEndWith, ".crt") - So(mockDatabaseConfigSSL.Username, ShouldEqual, "mockusername") - So(mockDatabaseConfigSSL.Password, ShouldEqual, "mockpassword") - So(mockDatabaseConfigSSL.Database, ShouldEqual, "mockname") - So(mockDatabaseConfigSSL.Host, ShouldEqual, "mockhost") - So(mockDatabaseConfigSSL.SSLMode, ShouldEqual, mockSSLModeVerifyCA) - So(mockDatabaseConfigSSL.QueryParams, ShouldEqual, map[string]string{"mockquery": "true"}) + _, err := ParseCFEnvs(&mockDatabaseConfigSSL, mockVarSet) + So(err, ShouldBeNil) + So(mockDatabaseConfigSSL.SSLRootCertificate, ShouldContainSubstring, "postgres-ssl-") + So(mockDatabaseConfigSSL.SSLRootCertificate, ShouldEndWith, ".crt") + So(mockDatabaseConfigSSL.Username, ShouldEqual, "mockusername") + So(mockDatabaseConfigSSL.Password, ShouldEqual, "mockpassword") + So(mockDatabaseConfigSSL.Database, ShouldEqual, "mockname") + So(mockDatabaseConfigSSL.Host, ShouldEqual, "mockhost") + So(mockDatabaseConfigSSL.SSLMode, ShouldEqual, mockSSLModeVerifyCA) + So(mockDatabaseConfigSSL.QueryParams, ShouldEqual, map[string]string{"mockquery": "true"}) + }) + + Convey("without query params present", func() { + mockEnvVarsMap["VCAP_SERVICES"] = `{"cf-postgresql-service": [ { "name": "mock-stratos-ssl", "credentials": { "cacrt": "mockcert", "host": "mockhost", "password": "mockpassword", "port": 5432, "username": "mockusername", "uri": "postgres://mockusername:mockpassword@mockhost:5432/mockname" } } ] }` + mockVarSet := env.NewVarSet(env.WithMapLookup(mockEnvVarsMap)) + + _, err := ParseCFEnvs(&mockDatabaseConfigSSL, mockVarSet) + So(err, ShouldBeNil) + So(mockDatabaseConfigSSL.SSLRootCertificate, ShouldContainSubstring, "postgres-ssl-") + So(mockDatabaseConfigSSL.SSLRootCertificate, ShouldEndWith, ".crt") + So(mockDatabaseConfigSSL.Username, ShouldEqual, "mockusername") + So(mockDatabaseConfigSSL.Password, ShouldEqual, "mockpassword") + So(mockDatabaseConfigSSL.Database, ShouldEqual, "mockname") + So(mockDatabaseConfigSSL.Host, ShouldEqual, "mockhost") + So(mockDatabaseConfigSSL.SSLMode, ShouldEqual, mockSSLModeVerifyCA) + So(mockDatabaseConfigSSL.QueryParams, ShouldEqual, map[string]string{}) + }) }) }) })