diff --git a/.gitignore b/.gitignore index 4cf6e279..c9766ca7 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,7 @@ audit-service/data/ # Allow .choreo directories for WSO2 Choreo deployment !**/.choreo/ !**/.choreo/** + +# Database data directories (local development) +exchange/pdp-data/ +exchange/ce-data/ diff --git a/exchange/consent-engine/main.go b/exchange/consent-engine/main.go index bff14b49..42c040ed 100644 --- a/exchange/consent-engine/main.go +++ b/exchange/consent-engine/main.go @@ -32,6 +32,17 @@ func main() { // Setup logging utils.SetupLogging(cfg.Logging.Format, cfg.Logging.Level) + // Initialize monitoring/observability (optional - can be disabled via ENABLE_OBSERVABILITY=false) + // Services will continue to function normally even if observability is disabled + if monitoring.IsObservabilityEnabled() { + monitoringConfig := monitoring.DefaultConfig("consent-engine") + if err := monitoring.Initialize(monitoringConfig); err != nil { + slog.Warn("Failed to initialize monitoring (service will continue)", "error", err) + } + } else { + slog.Info("Observability disabled via environment variable") + } + slog.Info("Starting consent engine", "environment", cfg.Environment, "port", cfg.Service.Port, diff --git a/exchange/docker-compose.yml b/exchange/docker-compose.yml index 39114d68..a8417c4c 100644 --- a/exchange/docker-compose.yml +++ b/exchange/docker-compose.yml @@ -98,4 +98,4 @@ services: networks: opendif-network: name: opendif-network - external: true \ No newline at end of file + external: true diff --git a/exchange/minimal-config.json b/exchange/minimal-config.json new file mode 100644 index 00000000..c1cca0f8 --- /dev/null +++ b/exchange/minimal-config.json @@ -0,0 +1,6 @@ +{ + "ceUrl": "http://consent-engine:8081", + "pdpUrl": "http://policy-decision-point:8082", + "providers": [], + "argMappings": [] +} diff --git a/exchange/orchestration-engine/provider/provider.go b/exchange/orchestration-engine/provider/provider.go index 7faa7f5d..e7ab8f13 100644 --- a/exchange/orchestration-engine/provider/provider.go +++ b/exchange/orchestration-engine/provider/provider.go @@ -6,9 +6,11 @@ import ( "fmt" "net/http" "sync" + "time" "github.com/ginaxu1/gov-dx-sandbox/exchange/orchestration-engine/logger" "github.com/ginaxu1/gov-dx-sandbox/exchange/orchestration-engine/pkg/auth" + "github.com/gov-dx-sandbox/exchange/shared/monitoring" "golang.org/x/oauth2/clientcredentials" ) @@ -47,7 +49,7 @@ func NewProvider(serviceKey, serviceUrl, schemaID string, authConfig *auth.AuthC } // PerformRequest performs the HTTP request to the provider with necessary authentication. -func (p *Provider) PerformRequest(ctx context.Context, reqBody []byte) (*http.Response, error) { +func (p *Provider) PerformRequest(ctx context.Context, reqBody []byte) (resp *http.Response, err error) { // 1. Create Request req, err := http.NewRequestWithContext(ctx, "POST", p.ServiceUrl, bytes.NewBuffer(reqBody)) if err != nil { @@ -56,21 +58,29 @@ func (p *Provider) PerformRequest(ctx context.Context, reqBody []byte) (*http.Re req.Header.Set("Content-Type", "application/json") + start := time.Now() + defer func() { + monitoring.RecordExternalCall(p.ServiceKey, "provider_request", time.Since(start), err) + }() + if p.Auth != nil { switch p.Auth.Type { case auth.AuthTypeOAuth2: if p.OAuth2Config == nil { - logger.Log.Error("OAuth2Config is nil", "providerKey", p.ServiceKey) - return nil, fmt.Errorf("OAuth2Config is nil") + err = fmt.Errorf("OAuth2Config is nil") + logger.Log.Error(err.Error(), "providerKey", p.ServiceKey) + return } client := p.OAuth2Config.Client(ctx) - return client.Do(req) // Use context with request + resp, err = client.Do(req) + return case auth.AuthTypeAPIKey: req.Header.Set(p.Auth.APIKeyName, p.Auth.APIKeyValue) } } // Default client execution (for API Key or no auth) - return p.Client.Do(req) + resp, err = p.Client.Do(req) + return } diff --git a/exchange/orchestration-engine/server/server.go b/exchange/orchestration-engine/server/server.go index 76cb3207..1bea0723 100644 --- a/exchange/orchestration-engine/server/server.go +++ b/exchange/orchestration-engine/server/server.go @@ -17,6 +17,7 @@ import ( "github.com/ginaxu1/gov-dx-sandbox/exchange/orchestration-engine/pkg/graphql" "github.com/ginaxu1/gov-dx-sandbox/exchange/orchestration-engine/services" "github.com/go-chi/chi/v5" + "github.com/gov-dx-sandbox/exchange/shared/monitoring" ) type Response struct { @@ -85,6 +86,7 @@ func RunServer(ctx context.Context, f *federator.Federator) { // SetupRouter initializes the router and registers all endpoints func SetupRouter(f *federator.Federator) *chi.Mux { mux := chi.NewRouter() + mux.Use(monitoring.HTTPMetricsMiddleware) // Initialize database connection dbConnectionString := getDatabaseConnectionString() @@ -123,6 +125,9 @@ func SetupRouter(f *federator.Federator) *chi.Mux { } }) + // Metrics endpoint + mux.Method("GET", "/metrics", monitoring.Handler()) + // Schema management routes mux.Get("/sdl", schemaHandler.GetActiveSchema) mux.Post("/sdl", schemaHandler.CreateSchema) @@ -135,6 +140,7 @@ func SetupRouter(f *federator.Federator) *chi.Mux { // Publicly accessible Endpoints mux.Post("/public/graphql", func(w http.ResponseWriter, r *http.Request) { + // Parse request body var req graphql.Request if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -181,6 +187,12 @@ func SetupRouter(f *federator.Federator) *chi.Mux { logger.Log.Error("Failed to write response", "error", err) return } + + outcome := "success" + if len(response.Errors) > 0 { + outcome = "failure" + } + monitoring.RecordBusinessEvent("graphql_request", outcome) }) return mux diff --git a/exchange/policy-decision-point/go.mod b/exchange/policy-decision-point/go.mod index 13ba55a5..81f83f38 100644 --- a/exchange/policy-decision-point/go.mod +++ b/exchange/policy-decision-point/go.mod @@ -4,6 +4,7 @@ go 1.24.6 require ( github.com/google/uuid v1.6.0 + github.com/gov-dx-sandbox/exchange/shared/monitoring v0.0.0 github.com/gov-dx-sandbox/exchange/shared/utils v0.0.0 github.com/stretchr/testify v1.10.0 gorm.io/driver/postgres v1.6.0 @@ -11,22 +12,52 @@ require ( gorm.io/gorm v1.31.0 ) -replace github.com/gov-dx-sandbox/exchange/shared/utils => ../shared/utils - require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.6.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/kr/pretty v0.3.1 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.60.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + go.opentelemetry.io/otel v1.32.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.54.0 // indirect + go.opentelemetry.io/otel/metric v1.32.0 // indirect + go.opentelemetry.io/otel/sdk v1.32.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect + go.opentelemetry.io/otel/trace v1.32.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.41.0 // indirect golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.27.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/gov-dx-sandbox/exchange/shared/config => ../shared/config + +replace github.com/gov-dx-sandbox/exchange/shared/constants => ../shared/constants + +replace github.com/gov-dx-sandbox/exchange/shared/utils => ../shared/utils + +replace github.com/gov-dx-sandbox/exchange/shared/monitoring => ../shared/monitoring diff --git a/exchange/policy-decision-point/go.sum b/exchange/policy-decision-point/go.sum index 07691979..74b9b294 100644 --- a/exchange/policy-decision-point/go.sum +++ b/exchange/policy-decision-point/go.sum @@ -1,9 +1,23 @@ -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 h1:ad0vkEBuk23VJzZR9nkLVG0YAoN9coASF1GusYX6AlU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0/go.mod h1:igFoXX2ELCW06bol23DWPB5BEWfZISOzSP5K2sbLea0= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -16,16 +30,28 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -33,12 +59,40 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= +go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 h1:t/Qur3vKSkUCcDVaSumWF2PKHt85pc7fRvFuoVT8qFU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0/go.mod h1:Rl61tySSdcOJWoEgYZVtmnKdA0GeKrSqkHC1t+91CH8= +go.opentelemetry.io/otel/exporters/prometheus v0.54.0 h1:rFwzp68QMgtzu9PgP3jm9XaMICI6TsofWWPcBDKwlsU= +go.opentelemetry.io/otel/exporters/prometheus v0.54.0/go.mod h1:QyjcV9qDP6VeK5qPyKETvNjmaaEc7+gqjh4SS0ZYzDU= +go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= +go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= +go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= +go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= +go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 h1:M0KvPgPmDZHPlbRbaNU1APr28TvwvvdUPlSv7PUvy8g= +google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:dguCy7UOdZhTvLzDyt15+rOrawrpM4q7DD9dQ1P11P4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 h1:XVhgTWWV3kGQlwJHR3upFWZeTsei6Oks1apkZSeonIE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/exchange/policy-decision-point/main.go b/exchange/policy-decision-point/main.go index f050a79a..bbb2c971 100644 --- a/exchange/policy-decision-point/main.go +++ b/exchange/policy-decision-point/main.go @@ -9,6 +9,7 @@ import ( "github.com/gov-dx-sandbox/exchange/policy-decision-point/internal/config" v1 "github.com/gov-dx-sandbox/exchange/policy-decision-point/v1" + "github.com/gov-dx-sandbox/exchange/shared/monitoring" "github.com/gov-dx-sandbox/exchange/shared/utils" ) @@ -26,6 +27,17 @@ func main() { // Setup logging utils.SetupLogging(cfg.Logging.Format, cfg.Logging.Level) + // Initialize monitoring/observability (optional - can be disabled via ENABLE_OBSERVABILITY=false) + // Services will continue to function normally even if observability is disabled + if monitoring.IsObservabilityEnabled() { + monitoringConfig := monitoring.DefaultConfig("policy-decision-point") + if err := monitoring.Initialize(monitoringConfig); err != nil { + slog.Warn("Failed to initialize monitoring (service will continue)", "error", err) + } + } else { + slog.Info("Observability disabled via environment variable") + } + slog.Info("Starting policy decision point", "environment", cfg.Environment, "port", cfg.Service.Port, @@ -78,6 +90,9 @@ func main() { mux := http.NewServeMux() v1Handler.SetupRoutes(mux) // V1 routes with /api/v1/policy/ prefix + // Metrics endpoint + mux.Handle("/metrics", monitoring.Handler()) + // Health check endpoint mux.Handle("/health", utils.PanicRecoveryMiddleware(utils.HealthHandler("policy-decision-point"))) @@ -151,6 +166,9 @@ func main() { utils.RespondWithJSON(w, http.StatusOK, debugInfo) }))) + // Wrap with metrics middleware + handler := monitoring.HTTPMetricsMiddleware(mux) + // Create server configuration serverConfig := &utils.ServerConfig{ Port: cfg.Service.Port, @@ -158,7 +176,7 @@ func main() { WriteTimeout: cfg.Service.Timeout, IdleTimeout: 60 * time.Second, } - server := utils.CreateServer(serverConfig, mux) + server := utils.CreateServer(serverConfig, handler) // Start server with graceful shutdown if err := utils.StartServerWithGracefulShutdown(server, "policy-decision-point"); err != nil { diff --git a/exchange/shared/monitoring/metrics.go b/exchange/shared/monitoring/metrics.go index 5b68761e..9b687ba7 100644 --- a/exchange/shared/monitoring/metrics.go +++ b/exchange/shared/monitoring/metrics.go @@ -1,6 +1,7 @@ package monitoring import ( + "errors" "log/slog" "net/http" "os" @@ -26,8 +27,22 @@ var ( // ensureInitialized ensures OpenTelemetry is initialized with default config // This is called automatically when metrics functions are used +// Observability can be disabled via ENABLE_OBSERVABILITY=false or OTEL_METRICS_ENABLED=false func ensureInitialized() { initOnce.Do(func() { + // Check if observability is explicitly disabled + enableObservability := getEnvBoolOrDefault("ENABLE_OBSERVABILITY", true) + otelMetricsEnabled := getEnvBoolOrDefault("OTEL_METRICS_ENABLED", true) + + if !enableObservability || !otelMetricsEnabled { + slog.Info("Observability disabled via environment variable, skipping initialization", + "ENABLE_OBSERVABILITY", enableObservability, + "OTEL_METRICS_ENABLED", otelMetricsEnabled) + // Use sentinel error so IsInitialized() returns false when observability is disabled + initErr = errors.New("observability disabled via environment variable") + return + } + // Try to get service name from environment or use default serviceName := os.Getenv("SERVICE_NAME") if serviceName == "" { @@ -61,6 +76,15 @@ func IsInitialized() bool { return initErr == nil } +// IsObservabilityEnabled checks if observability is enabled via environment variables +// Returns false if ENABLE_OBSERVABILITY=false or OTEL_METRICS_ENABLED=false +// Returns true otherwise (default behavior) +func IsObservabilityEnabled() bool { + enableObservability := getEnvBoolOrDefault("ENABLE_OBSERVABILITY", true) + otelMetricsEnabled := getEnvBoolOrDefault("OTEL_METRICS_ENABLED", true) + return enableObservability && otelMetricsEnabled +} + // RegisterRoutes registers routes for normalization. Supports static routes and templates with :id or {id} placeholders. // Templates match incoming paths and normalize dynamic segments. Call during service initialization. // diff --git a/exchange/shared/monitoring/metrics_test.go b/exchange/shared/monitoring/metrics_test.go index b194a125..30dfe6a8 100644 --- a/exchange/shared/monitoring/metrics_test.go +++ b/exchange/shared/monitoring/metrics_test.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" "strings" "testing" "time" @@ -427,6 +428,85 @@ func TestMultipleInitializations(t *testing.T) { } } +// TestIsObservabilityEnabled tests the IsObservabilityEnabled function +func TestIsObservabilityEnabled(t *testing.T) { + tests := []struct { + name string + enableObservability string + otelMetricsEnabled string + expected bool + }{ + { + name: "Both enabled (default)", + enableObservability: "", + otelMetricsEnabled: "", + expected: true, + }, + { + name: "ENABLE_OBSERVABILITY=false disables", + enableObservability: "false", + otelMetricsEnabled: "", + expected: false, + }, + { + name: "OTEL_METRICS_ENABLED=false disables", + enableObservability: "", + otelMetricsEnabled: "false", + expected: false, + }, + { + name: "Both false disables", + enableObservability: "false", + otelMetricsEnabled: "false", + expected: false, + }, + { + name: "Both true enables", + enableObservability: "true", + otelMetricsEnabled: "true", + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Save original values + origEnable := os.Getenv("ENABLE_OBSERVABILITY") + origOtel := os.Getenv("OTEL_METRICS_ENABLED") + + // Set test values + if tt.enableObservability != "" { + os.Setenv("ENABLE_OBSERVABILITY", tt.enableObservability) + } else { + os.Unsetenv("ENABLE_OBSERVABILITY") + } + if tt.otelMetricsEnabled != "" { + os.Setenv("OTEL_METRICS_ENABLED", tt.otelMetricsEnabled) + } else { + os.Unsetenv("OTEL_METRICS_ENABLED") + } + + // Test (function doesn't depend on initOnce, so no reset needed) + result := IsObservabilityEnabled() + if result != tt.expected { + t.Errorf("IsObservabilityEnabled() = %v, want %v", result, tt.expected) + } + + // Restore original values + if origEnable != "" { + os.Setenv("ENABLE_OBSERVABILITY", origEnable) + } else { + os.Unsetenv("ENABLE_OBSERVABILITY") + } + if origOtel != "" { + os.Setenv("OTEL_METRICS_ENABLED", origOtel) + } else { + os.Unsetenv("OTEL_METRICS_ENABLED") + } + }) + } +} + // TestHTTPMetricsMiddlewareWithDifferentStatusCodes tests that different HTTP status codes are recorded func TestHTTPMetricsMiddlewareWithDifferentStatusCodes(t *testing.T) { testCases := []struct { diff --git a/exchange/shared/monitoring/otel_metrics.go b/exchange/shared/monitoring/otel_metrics.go index 12d18acf..8af22442 100644 --- a/exchange/shared/monitoring/otel_metrics.go +++ b/exchange/shared/monitoring/otel_metrics.go @@ -87,9 +87,28 @@ type Config struct { } // DefaultConfig returns a default configuration +// Observability can be disabled by setting ENABLE_OBSERVABILITY=false or OTEL_METRICS_ENABLED=false func DefaultConfig(serviceName string) Config { + // Check if observability is explicitly disabled + enableObservability := getEnvBoolOrDefault("ENABLE_OBSERVABILITY", true) + otelMetricsEnabled := getEnvBoolOrDefault("OTEL_METRICS_ENABLED", true) + + // If either flag is false, disable observability + observabilityEnabled := enableObservability && otelMetricsEnabled + + var exporterType string + if !observabilityEnabled { + exporterType = "none" + slog.Info("Observability disabled via environment variable", + "service", serviceName, + "ENABLE_OBSERVABILITY", enableObservability, + "OTEL_METRICS_ENABLED", otelMetricsEnabled) + } else { + exporterType = getEnvOrDefault("OTEL_METRICS_EXPORTER", "prometheus") + } + return Config{ - ExporterType: getEnvOrDefault("OTEL_METRICS_EXPORTER", "prometheus"), + ExporterType: exporterType, ServiceName: serviceName, ServiceVersion: getEnvOrDefault("SERVICE_VERSION", "dev"), OTLPEndpoint: getEnvOrDefault("OTEL_EXPORTER_OTLP_ENDPOINT", ""), diff --git a/observability/README.md b/observability/README.md index bb6f3635..cc248b71 100644 --- a/observability/README.md +++ b/observability/README.md @@ -79,6 +79,24 @@ Collects real-time metrics from all Go services for debugging performance and er ## Quick Start +### Automated Setup and Verification + +Run the verification script to start everything and verify metrics are working: + +```bash +cd observability +./verify_observability_setup.sh +``` + +This script will: +1. Start Prometheus and Grafana +2. Start required services (Orchestration Engine, Policy Decision Point) with database +3. Verify metrics endpoints are accessible +4. Generate test traffic +5. Check Prometheus targets and Grafana accessibility + +### Manual Setup + ```bash cd observability docker compose up -d diff --git a/observability/grafana/dashboards/go-services-metrics.json b/observability/grafana/dashboards/go-services-metrics.json index bd00c545..b0c2dcf5 100644 --- a/observability/grafana/dashboards/go-services-metrics.json +++ b/observability/grafana/dashboards/go-services-metrics.json @@ -16,7 +16,6 @@ "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, - "uid": "go-services-dashboard", "iteration": 1710643200000, "links": [], "panels": [ @@ -36,8 +35,8 @@ }, "targets": [ { - "expr": "sum(rate(http_requests_total[5m])) by (method, route)", - "legendFormat": "{{method}} {{route}}", + "expr": "sum(rate(http_requests_total[5m])) by (http_request_method, http_route)", + "legendFormat": "{{http_request_method}} {{http_route}}", "refId": "A" } ], @@ -61,8 +60,8 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (route, le))", - "legendFormat": "{{route}}", + "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (http_route, le))", + "legendFormat": "{{http_route}}", "refId": "A" } ], diff --git a/observability/grafana/provisioning/dashboards/dashboard.yml b/observability/grafana/provisioning/dashboards/dashboard.yml index 549fd8c8..9304fb75 100644 --- a/observability/grafana/provisioning/dashboards/dashboard.yml +++ b/observability/grafana/provisioning/dashboards/dashboard.yml @@ -1,13 +1,12 @@ apiVersion: 1 providers: - - name: "Go Services Dashboards" + - name: Go Services Dashboards orgId: 1 folder: "" type: file disableDeletion: false - updateIntervalSeconds: 10 - allowUiUpdates: true + editable: true options: path: /var/lib/grafana/dashboards foldersFromFilesStructure: true diff --git a/observability/grafana/provisioning/datasources/datasource.yml b/observability/grafana/provisioning/datasources/datasource.yml index d558bb15..8ce2707f 100644 --- a/observability/grafana/provisioning/datasources/datasource.yml +++ b/observability/grafana/provisioning/datasources/datasource.yml @@ -7,8 +7,5 @@ datasources: orgId: 1 url: http://prometheus:9090 isDefault: true - editable: false - jsonData: - httpMethod: POST - timeInterval: 10s + editable: true diff --git a/observability/prometheus/prometheus.yml b/observability/prometheus/prometheus.yml index 0e265e82..dcbc6cbb 100644 --- a/observability/prometheus/prometheus.yml +++ b/observability/prometheus/prometheus.yml @@ -35,13 +35,13 @@ scrape_configs: port: '8082' # Root Level Services - - job_name: portal-backend + - job_name: api-server-go metrics_path: /metrics static_configs: - targets: - - portal-backend:3000 + - api-server-go:3000 labels: - service: 'portal-backend' + service: 'api-server-go' port: '3000' - job_name: audit-service @@ -52,4 +52,3 @@ scrape_configs: labels: service: 'audit-service' port: '3001' - diff --git a/observability/verify_observability_setup.sh b/observability/verify_observability_setup.sh new file mode 100755 index 00000000..d624af49 --- /dev/null +++ b/observability/verify_observability_setup.sh @@ -0,0 +1,83 @@ +#!/bin/bash +set -e + +# Color codes +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log() { echo -e "${BLUE}==>${NC} $1"; } +error() { echo -e "${RED}❌ $1${NC}"; exit 1; } +success() { echo -e "${GREEN}✅ $1${NC}"; } +warn() { echo -e "${YELLOW}⚠️ $1${NC}"; } + +echo "==========================================" +echo " Observability Setup" +echo "==========================================" + +# Check Docker +docker info > /dev/null 2>&1 || error "Docker is not running" + +# Create network if needed +docker network create opendif-network 2>/dev/null || true + +# Start observability stack +log "Starting Prometheus & Grafana..." +cd "$(dirname "$0")" +docker compose up -d + +# Start exchange services +log "Starting exchange services..." +cd ../exchange +export DB_PASSWORD=${DB_PASSWORD:-password} +export DB_HOST=${DB_HOST:-localhost} # Required for orchestration-engine + +# Start databases +docker compose up -d pdp-db ce-db +log "Waiting for databases..." +sleep 5 + +# Start services +docker compose up -d orchestration-engine policy-decision-point consent-engine +log "Waiting for services to start..." +sleep 10 + +# Generate some traffic +log "Generating traffic..." +for i in {1..3}; do + curl -s http://localhost:4000/health > /dev/null || true + curl -s http://localhost:8081/health > /dev/null || true + curl -s http://localhost:8082/health > /dev/null || true + sleep 1 +done + +# Step 5: Verify Prometheus Scraping +log "Step 5: Checking Prometheus target health..." +if command -v jq > /dev/null 2>&1; then + TARGETS=$(curl -s http://localhost:9091/api/v1/targets) + # Filter for targets that are 'up' + HEALTHY_COUNT=$(echo "$TARGETS" | jq '.data.activeTargets[] | select(.health=="up") | .health' | wc -l) + TOTAL_COUNT=$(echo "$TARGETS" | jq '.data.activeTargets[] | .health' | wc -l) + + if [ "$HEALTHY_COUNT" -eq "$TOTAL_COUNT" ] && [ "$TOTAL_COUNT" -gt 0 ]; then + success "Prometheus is successfully scraping all $TOTAL_COUNT targets." + else + warn "Prometheus targets: $HEALTHY_COUNT/$TOTAL_COUNT are UP." + echo "$TARGETS" | jq -r '.data.activeTargets[] | "[\(.health)] \(.labels.job) -> \(.scrapeUrl)"' + fi +else + warn "Install 'jq' for detailed Prometheus target verification." + curl -s http://localhost:9091/api/v1/targets | head -n 5 +fi + +echo "" +echo "==========================================" +success "Setup Complete!" +echo "==========================================" +echo "" +echo "Prometheus Targets: http://localhost:9091/targets" +echo "Prometheus Graph: http://localhost:9091/graph" +echo "Grafana Dashboard: http://localhost:3002/d/go-services-dashboard/go-services-metrics" +echo ""