From 7c688fffae74268b68199daaab0e36446799eb91 Mon Sep 17 00:00:00 2001 From: Gregory Schofield Date: Thu, 10 Jul 2025 13:20:33 -0400 Subject: [PATCH 1/3] Update default locations for configs and data. Signed-off-by: Gregory Schofield --- internal/config/config.go | 73 +++++++++++-------- internal/db/db.go | 4 +- .../middleware/{gemfast_acl.csv => acl.csv} | 0 internal/middleware/acl.go | 24 ++---- .../{auth_model.conf => model.conf} | 0 5 files changed, 54 insertions(+), 47 deletions(-) rename internal/middleware/{gemfast_acl.csv => acl.csv} (100%) rename internal/middleware/{auth_model.conf => model.conf} (100%) diff --git a/internal/config/config.go b/internal/config/config.go index 8128adf..7293139 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -5,7 +5,10 @@ import ( "net/url" "os" "os/user" + "path/filepath" + "runtime" + "github.com/gemfast/server/internal/utils" "github.com/hashicorp/hcl/v2/hclsimple" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -13,22 +16,22 @@ import ( ) type Config struct { - Port int `hcl:"port,optional"` - LogLevel string `hcl:"log_level,optional"` - Dir string `hcl:"dir,optional"` - GemDir string `hcl:"gem_dir,optional"` - DBDir string `hcl:"db_dir,optional"` - ACLPath string `hcl:"acl_path,optional"` - AuthModelPath string `hcl:"auth_model_path,optional"` - PrivateGemsNamespace string `hcl:"private_gems_namespace,optional"` - UIDisabled bool `hcl:"ui_disabled,optional"` - MetricsDisabled bool `hcl:"metrics_disabled,optional"` + user *user.User `hcl:"-"` + Port int `hcl:"port,optional"` + LogLevel string `hcl:"log_level,optional"` + Dir string `hcl:"dir,optional"` + GemDir string `hcl:"gem_dir,optional"` + DBDir string `hcl:"db_dir,optional"` + ACLPath string `hcl:"acl_path,optional"` + AuthModelPath string `hcl:"auth_model_path,optional"` + PrivateGemsNamespace string `hcl:"private_gems_namespace,optional"` + UIDisabled bool `hcl:"ui_disabled,optional"` + MetricsDisabled bool `hcl:"metrics_disabled,optional"` - LicenseKey string `hcl:"license_key,optional"` - Mirrors []*MirrorConfig `hcl:"mirror,block"` - Filter *FilterConfig `hcl:"filter,block"` - CVE *CVEConfig `hcl:"cve,block"` - Auth *AuthConfig `hcl:"auth,block"` + Mirrors []*MirrorConfig `hcl:"mirror,block"` + Filter *FilterConfig `hcl:"filter,block"` + CVE *CVEConfig `hcl:"cve,block"` + Auth *AuthConfig `hcl:"auth,block"` } type MirrorConfig struct { @@ -70,15 +73,16 @@ type LocalUser struct { } func NewConfig() *Config { - cfg := Config{} + usr, err := user.Current() + if err != nil { + log.Warn().Err(err).Msg("unable to get the current linux user") + } + cfg := Config{user: usr} cfgFile := os.Getenv("GEMFAST_CONFIG_FILE") if cfgFile == "" { cfgFileTries := []string{"/etc/gemfast/gemfast.hcl"} - usr, err := user.Current() - if err != nil { - log.Warn().Err(err).Msg("unable to get the current linux user") - } else { - cfgFileTries = append(cfgFileTries, fmt.Sprintf("%s/.gemfast/gemfast.hcl", usr.HomeDir)) + if usr != nil { + cfgFileTries = append(cfgFileTries, fmt.Sprintf("%s/.config/gemfast/gemfast.hcl", usr.HomeDir)) } for _, f := range cfgFileTries { if _, err := os.Stat(f); err == nil { @@ -95,7 +99,7 @@ func NewConfig() *Config { return &cfg } } - err := hclsimple.DecodeFile(cfgFile, nil, &cfg) + err = hclsimple.DecodeFile(cfgFile, nil, &cfg) if err != nil { log.Error().Err(err).Msg(fmt.Sprintf("failed to load configuration file %s", cfgFile)) os.Exit(1) @@ -121,7 +125,16 @@ func (c *Config) setDefaultServerConfig() { } configureLogLevel(c.LogLevel) if c.Dir == "" { - c.Dir = "/var/gemfast" + if c.user.Username == "root" { + c.Dir = "/var/lib/gemfast/data" + } else if runtime.GOOS == "darwin" { + c.Dir = fmt.Sprintf("%s/Library/Application Support/gemfast", c.user.HomeDir) + } else if runtime.GOOS == "linux" { + c.Dir = fmt.Sprintf("%s/gemfast", os.Getenv("XDG_DATA_HOME")) + if c.Dir == "/gemfast" { + c.Dir = fmt.Sprintf("%s/.local/share/gemfast", c.user.HomeDir) + } + } } if c.GemDir == "" { c.GemDir = fmt.Sprintf("%s/gems", c.Dir) @@ -174,13 +187,15 @@ func readJWTSecretKeyFromPath(keyPath string) string { log.Info().Msg("generating a new JWT secret key") pw, err := password.Generate(64, 10, 0, false, true) if err != nil { - log.Error().Err(err).Msg("unable to generate a new jwt secret key") - os.Exit(1) + log.Fatal().Err(err).Msg("unable to generate a new jwt secret key") + } + err = utils.MkDirs(filepath.Dir(keyPath)) + if err != nil { + log.Fatal().Err(err).Msg("unable to create directory for JWT secret key file") } file, err := os.Create(keyPath) if err != nil { - log.Error().Err(err).Msg("unable to create JWT secret key file") - os.Exit(1) + log.Fatal().Err(err).Msg("unable to create JWT secret key file") } defer file.Close() _, err = file.WriteString(pw) @@ -193,7 +208,7 @@ func readJWTSecretKeyFromPath(keyPath string) string { } func (c *Config) setDefaultAuthConfig() { - defaultJWTSecretKeyPath := ".jwt_secret_key" + defaultJWTSecretKeyPath := fmt.Sprintf("%s/.jwt_secret_key", c.Dir) if c.Auth == nil { c.Auth = &AuthConfig{ Type: "local", @@ -252,6 +267,6 @@ func (c *Config) setDefaultCVEConfig() { c.CVE.MaxSeverity = "high" } if c.CVE.RubyAdvisoryDBDir == "" { - c.CVE.RubyAdvisoryDBDir = "ruby-advisory-db" + c.CVE.RubyAdvisoryDBDir = fmt.Sprintf("%s/ruby-advisory-db", c.Dir) } } diff --git a/internal/db/db.go b/internal/db/db.go index 5bf9534..e536685 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -3,11 +3,11 @@ package db import ( "fmt" "net/http" - "os" "path/filepath" "strconv" "github.com/gemfast/server/internal/config" + "github.com/gemfast/server/internal/utils" "github.com/rs/zerolog/log" "go.etcd.io/bbolt" bolt "go.etcd.io/bbolt" @@ -30,7 +30,7 @@ func NewTestDB(boltDB *bolt.DB, cfg *config.Config) *DB { } func NewDB(cfg *config.Config) (*DB, error) { - err := os.MkdirAll(cfg.DBDir, os.ModePerm) + err := utils.MkDirs(cfg.DBDir) if err != nil { log.Logger.Error().Err(err).Msg(fmt.Sprintf("failed to create db directory %s", cfg.DBDir)) return nil, err diff --git a/internal/middleware/gemfast_acl.csv b/internal/middleware/acl.csv similarity index 100% rename from internal/middleware/gemfast_acl.csv rename to internal/middleware/acl.csv diff --git a/internal/middleware/acl.go b/internal/middleware/acl.go index d33e541..902f469 100644 --- a/internal/middleware/acl.go +++ b/internal/middleware/acl.go @@ -11,10 +11,10 @@ import ( "github.com/rs/zerolog/log" ) -//go:embed auth_model.conf +//go:embed model.conf var embeddedAuthModel []byte -//go:embed gemfast_acl.csv +//go:embed acl.csv var embeddedACL []byte type ACL struct { @@ -33,7 +33,8 @@ func NewACL(cfg *config.Config) *ACL { log.Fatal().Err(err).Msg("failed to get absolute path for acl") } } else { - policyPath, err = writeTempFile("gemfast_acl.csv", embeddedACL) + policyPath = fmt.Sprintf("%s/acl.csv", cfg.Dir) + err = os.WriteFile(policyPath, embeddedACL, 0644) if err != nil { log.Fatal().Err(err).Msg("failed to write embedded acl to temp file") } @@ -45,34 +46,25 @@ func NewACL(cfg *config.Config) *ACL { log.Fatal().Err(err).Msg("failed to get absolute path for auth_model") } } else { - authPath, err = writeTempFile("auth_model.conf", embeddedAuthModel) + authPath = fmt.Sprintf("%s/model.conf", cfg.Dir) + err = os.WriteFile(authPath, embeddedAuthModel, 0644) if err != nil { log.Fatal().Err(err).Msg("failed to write embedded auth model to temp file") } } if policyPath == "" || authPath == "" { - log.Fatal().Err(fmt.Errorf("unable to locate auth_model and gemfast_acl")).Msg("failed to find acl files") + log.Fatal().Err(fmt.Errorf("unable to locate model.conf and acl.csv")).Msg("failed to find acl files") } acl, err := casbin.NewEnforcer(authPath, policyPath) if err != nil { log.Fatal().Err(err).Msg("failed to initialize the acl") } - log.Info().Str("detail", policyPath).Msg("successfully initialized ACL enforcer") + log.Info().Str("detail", policyPath).Str("detail", authPath).Msg("successfully initialized ACL enforcer") return &ACL{casbin: acl, cfg: cfg} } -func writeTempFile(name string, content []byte) (string, error) { - tmp, err := os.CreateTemp("", name) - if err != nil { - return "", err - } - defer tmp.Close() - _, err = tmp.Write(content) - return tmp.Name(), err -} - func (acl *ACL) Enforce(role string, path string, method string) (bool, error) { return acl.casbin.Enforce(role, path, method) } diff --git a/internal/middleware/auth_model.conf b/internal/middleware/model.conf similarity index 100% rename from internal/middleware/auth_model.conf rename to internal/middleware/model.conf From ebc600686e1f5c7a0470b48dae1f446b26fc0d58 Mon Sep 17 00:00:00 2001 From: Gregory Schofield Date: Thu, 10 Jul 2025 13:20:55 -0400 Subject: [PATCH 2/3] Update readme to reflect the new default locations. Signed-off-by: Gregory Schofield --- README.md | 105 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index afc9f9b..20379eb 100644 --- a/README.md +++ b/README.md @@ -33,14 +33,14 @@ Gemfast is currently distributed in two different ways, a `docker` image and pre When running Gemfast as a container, its important to mount the following directories: -* /var/gemfast - The directory for the Gemfast data including gems and database -* /etc/gemfast - The directory for the gemfast.hcl config file. It is possible to configure the config file path using `env GEMFAST_CONFIG_FILE=/path/to/my/file.hcl` +* /var/lib/gemfast/data - The directory for the Gemfast data including gems and database +* /etc/gemfast - The directory for the gemfast.hcl config file ```bash docker run -d --name gemfast-server \ -p 2020:2020 \ - -v /etc/gemfast:/etc/gemfast \ - -v /var/gemfast:/var/gemfast \ + -v ./gemfast.hcl:/etc/gemfast/gemfast.hcl:ro \ + -v ./data:/var/lib/gemfast/data \ ghcr.io/gemfast/server:latest ``` @@ -63,11 +63,102 @@ make ./bin/gemfast-server ``` -## Docs +## Configuration -You can configure gemfast settings using the `/etc/gemfast/gemfast.hcl` file. There are many options all of which are listed in the documentation. +### Configuration File -For more information see: https://gemfast.io/docs/configuration/ +Gemfast is configured using an HCL file. You can customize the location of this file using the `$GEMFAST_CONFIG_FILE` environment variable or by passing the `--config` argument when starting the server. + +The places Gemfast automatically checks for configuration files are: `["/etc/gemfast/gemfast.hcl", "~/.config/gemfast/gemfast.hcl"]` + +### Configuration Options + +```terraform +# ========== gemfast.hcl ========== + +# Port to bind the HTTP server to. Defaults to 2020. +port = 2020 + +# Log level (trace, debug, info, warn, error, fatal, panic) +log_level = "info" + +# Base data directory for gemfast. If not set, defaults to platform-specific user data dir. +dir = "/var/lib/gemfast/data" + +# Directory to store downloaded gem files. +gem_dir = "/var/lib/gemfast/data/gems" + +# Directory to store SQLite database files. +db_dir = "/var/lib/gemfast/data/db" + +# Optional path to an ACL file (Casbin policy). +acl_path = "/var/lib/gemfast/data/acl.csv" + +# Optional path to an authorization model file (Casbin model). +auth_model_path = "/var/lib/gemfast/data/model.conf" + +# Namespace prefix for private gems (default is "private"). +private_gems_namespace = "private" + +# Disable the web UI if true. +ui_disabled = false + +# Disable Prometheus metrics endpoint if true. +metrics_disabled = false + +# ==== Mirror block ==== +# Define external sources to mirror gems from. +mirror "https://rubygems.org" { + enabled = true # Set to false to disable this mirror + # hostname is auto-derived from upstream, but can be overridden + # hostname = "rubygems.org" +} + +# ==== Filter block ==== +# Configure regex-based gem allow/deny logic. +filter { + enabled = true # Enable filtering + action = "deny" # Action can be "allow" or "deny" + regex = ["^evil-.*", "^bad-gem$"] # Regex list for filtering gem names +} + +# ==== CVE block ==== +# Control Ruby CVE integration. +cve { + enabled = true # Enable CVE scanning + max_severity = "high" # Only block gems above this severity + ruby_advisory_db_dir = "/var/lib/gemfast/data/ruby-advisory-db" # Directory to store the CVE DB +} + +# ==== Auth block ==== +# Configure authentication settings. You can only specify a single auth block. +auth "local" { # can be local, github, or none + bcrypt_cost = 10 # bcrypt cost for hashing passwords + allow_anonymous_read = false # Allow unauthenticated read access + default_user_role = "read" # Default role for newly created users + + # If no users are specified, a default admin user will be created and the password written to the logs + user { # repeat this block to add more users + username = "admin" + password = "changeme" + role = "admin" + } + + # JWT secret used to sign access tokens (you can provide one or generate it automatically) + secret_key_path = "/var/lib/gemfast/data/.jwt_secret_key" +} + +# GitHub OAuth integration +# auth "github" { + + # github_client_id = "" + # github_client_secret = "" + # github_user_orgs = ["my-org"] # Restrict access to users in these GitHub orgs +# } + +# No auth +# auth "none" {} +``` ## UI From 2515f8f7440f140fd2acdf5cf4a1f1d9b07c7bed Mon Sep 17 00:00:00 2001 From: Gregory Schofield Date: Sat, 12 Jul 2025 15:59:33 -0400 Subject: [PATCH 3/3] Remove license from test configs and update data path. Signed-off-by: Gregory Schofield --- scripts/_functions.sh | 2 +- scripts/run_auth_tests.sh | 2 -- scripts/run_cve_tests.sh | 1 - scripts/run_filter_tests.sh | 2 -- scripts/run_private_gem_tests.sh | 2 -- scripts/run_smoke_tests.sh | 11 +++++------ scripts/upload_all.sh | 2 +- 7 files changed, 7 insertions(+), 15 deletions(-) diff --git a/scripts/_functions.sh b/scripts/_functions.sh index 0f2d6f5..7fc1d58 100644 --- a/scripts/_functions.sh +++ b/scripts/_functions.sh @@ -4,7 +4,7 @@ function start_server() { local BUILD_TYPE="$1" if [[ "$BUILD_TYPE" == "docker" ]]; then docker load -i gemfast*.tar - docker run -d --name gemfast-server -p 2020:2020 -v /etc/gemfast:/etc/gemfast -v /var/gemfast:/var/gemfast goreleaser.ko.local/server:latest start + docker run -d --name gemfast-server -p 2020:2020 -v /etc/gemfast:/etc/gemfast -v ./data:/var/lib/gemfast/data goreleaser.ko.local/server:latest start sleep 5 docker ps docker logs gemfast-server diff --git a/scripts/run_auth_tests.sh b/scripts/run_auth_tests.sh index 799282d..678a1f5 100755 --- a/scripts/run_auth_tests.sh +++ b/scripts/run_auth_tests.sh @@ -14,7 +14,6 @@ sudo mkdir -p /var/gemfast sudo chown -R $USER: /etc/gemfast sudo chown -R $USER: /var/gemfast cat << CONFIG > /etc/gemfast/gemfast.hcl -license_key = "B7D865-DA12D3-11DA3D-DD81AE-9420D3-V3" auth "local" { allow_anonymous_read = false admin_password = "foobar" @@ -57,7 +56,6 @@ for gem in *.gem; do done sleep 5 -sudo ls -la /var/gemfast/gems sudo rm -f Gemfile Gemfile.lock cat << CONFIG > Gemfile source "https://rubygems.org" diff --git a/scripts/run_cve_tests.sh b/scripts/run_cve_tests.sh index 91dbe4b..8873b1f 100755 --- a/scripts/run_cve_tests.sh +++ b/scripts/run_cve_tests.sh @@ -12,7 +12,6 @@ gem update --system sudo mkdir -p /etc/gemfast sudo chown -R $USER: /etc/gemfast cat << CONFIG > /etc/gemfast/gemfast.hcl -license_key = "B7D865-DA12D3-11DA3D-DD81AE-9420D3-V3" auth "none" {} cve { enabled = true diff --git a/scripts/run_filter_tests.sh b/scripts/run_filter_tests.sh index c8ea72a..0344ccf 100755 --- a/scripts/run_filter_tests.sh +++ b/scripts/run_filter_tests.sh @@ -12,7 +12,6 @@ gem update --system sudo mkdir -p /etc/gemfast sudo chown -R $USER: /etc/gemfast cat << CONFIG > /etc/gemfast/gemfast.hcl -license_key = "B7D865-DA12D3-11DA3D-DD81AE-9420D3-V3" auth "none" {} filter { enabled = true @@ -38,7 +37,6 @@ popd sudo rm -rf /etc/gemfast/gemfast.hcl sudo tee /etc/gemfast/gemfast.hcl > /dev/null <<'CONFIG' -license_key = "B7D865-DA12D3-11DA3D-DD81AE-9420D3-V3" auth "none" {} filter { enabled = true diff --git a/scripts/run_private_gem_tests.sh b/scripts/run_private_gem_tests.sh index ba09209..4cebaeb 100755 --- a/scripts/run_private_gem_tests.sh +++ b/scripts/run_private_gem_tests.sh @@ -12,7 +12,6 @@ gem update --system sudo mkdir -p /etc/gemfast sudo chown -R $USER: /etc/gemfast cat << CONFIG > /etc/gemfast/gemfast.hcl -license_key = "B7D865-DA12D3-11DA3D-DD81AE-9420D3-V3" auth "none" {} private_gems_namespace = "foobar" CONFIG @@ -43,7 +42,6 @@ for gem in *.gem; do done sleep 5 -sudo ls -la /var/gemfast/gems sudo rm -f Gemfile Gemfile.lock cat << CONFIG > Gemfile source "https://rubygems.org" diff --git a/scripts/run_smoke_tests.sh b/scripts/run_smoke_tests.sh index 80b7334..079df1e 100755 --- a/scripts/run_smoke_tests.sh +++ b/scripts/run_smoke_tests.sh @@ -12,26 +12,25 @@ gem update --system sudo mkdir -p /etc/gemfast sudo chown -R $USER: /etc/gemfast cat << CONFIG > /etc/gemfast/gemfast.hcl -license_key = "B7D865-DA12D3-11DA3D-DD81AE-9420D3-V3" auth "none" {} CONFIG start_server "$BUILD_TYPE" -cd ./clones +pushd ./clones/rails -pushd rails bundle config mirror.https://rubygems.org http://localhost:2020 bundle +popd numGems=$(curl -s http://localhost:2020/admin/api/v1/stats/bucket | jq -r '.gems.KeyN') curl -s http://localhost:2020/admin/api/v1/backup > gemfast.db -sudo rm -rf /var/gemfast/db/gemfast.db -sudo mv ./gemfast.db /var/gemfast/db/gemfast.db +sudo rm -f ./data/db/gemfast.db +sudo mv ./gemfast.db ./data/db/gemfast.db if [ "$BUILD_TYPE" != "docker" ]; then - sudo chown gemfast: /var/gemfast/db/gemfast.db + sudo chown gemfast: ./data/db/gemfast.db fi restart_server "$BUILD_TYPE" diff --git a/scripts/upload_all.sh b/scripts/upload_all.sh index 5ec5af1..72d678d 100755 --- a/scripts/upload_all.sh +++ b/scripts/upload_all.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -gems_dir="/var/gemfast/gems" +gems_dir="./data/gems" cd $gems_dir for gem in *.gem; do [ -f "$gem" ] || break