From a150f94862858c76647c3ca2d338c34ae58207d3 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 15 Oct 2025 16:45:34 +0200 Subject: [PATCH 01/13] add devcontainer with postgresql and mysql support - initial devcontainer configuration - postgres and mysql database initialization scripts - database startup scripts and permission fixes - password and connectivity configuration --- .devcontainer/devcontainer.json | 43 ++++++++++++++++++++++++++ .devcontainer/init-mysql.sh | 54 +++++++++++++++++++++++++++++++++ .devcontainer/start-mysql.sh | 29 ++++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 .devcontainer/devcontainer.json create mode 100755 .devcontainer/init-mysql.sh create mode 100755 .devcontainer/start-mysql.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..64b5b784 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,43 @@ +{ + "name": "singularity", + "image": "mcr.microsoft.com/devcontainers/go:1.24", + "customizations": { + "vscode": { + "extensions": [ + "golang.go", + "eamodio.gitlens" + ], + "settings": { + "go.useLanguageServer": true, + "go.toolsEnvVars": { + "GOPATH": "/home/vscode/go" + } + } + } + }, + "features": { + "ghcr.io/devcontainers/features/go:1": { + "version": "1.24" + }, + "ghcr.io/itsmechlark/features/postgresql:1": { + "version": "16" + }, + "ghcr.io/devcontainers-extra/features/mysql-homebrew:1": {} + }, + "forwardPorts": [], + "postCreateCommand": "go mod download", + "postStartCommand": "/bin/bash -lc 'echo \"local all all trust\nhost all all 127.0.0.1/32 trust\nhost all all ::1/128 trust\" | sudo tee /etc/postgresql/16/main/pg_hba.conf > /dev/null && sudo service postgresql restart && /workspace/.devcontainer/start-mysql.sh && /workspace/.devcontainer/init-mysql.sh'", + "remoteUser": "vscode", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached,relabel=private", + "workspaceFolder": "/workspace", + "containerEnv": { + "GOPATH": "/home/vscode/go", + "MYSQL_DATABASE": "singularity", + "MYSQL_USER": "singularity", + "MYSQL_PASSWORD": "singularity", + "MYSQL_SOCKET": "/tmp/mysql.sock" + }, + "runArgs": ["--userns=keep-id"], + "containerUser": "vscode" +} + diff --git a/.devcontainer/init-mysql.sh b/.devcontainer/init-mysql.sh new file mode 100755 index 00000000..0f0a7265 --- /dev/null +++ b/.devcontainer/init-mysql.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +# If mysql client isn't present, exit quietly +if ! command -v mysql >/dev/null 2>&1; then + exit 0 +fi + +# Use same socket as mysql.server (homebrew default) +SOCKET="${MYSQL_SOCKET:-/tmp/mysql.sock}" + +# One-time init guard +MARKER="/workspace/.devcontainer/.mysql-initialized" +if [ -f "$MARKER" ]; then + exit 0 +fi + +# Determine root auth flags +MYSQL_ROOT_FLAGS=("-uroot") +if [ -n "${MYSQL_ROOT_PASSWORD:-}" ]; then + MYSQL_ROOT_FLAGS+=("-p${MYSQL_ROOT_PASSWORD}") +fi + +# Wait for server readiness (best effort) +for i in {1..60}; do + if mysqladmin --socket="$SOCKET" ping "${MYSQL_ROOT_FLAGS[@]}" >/dev/null 2>&1; then + break + fi + sleep 1 +done + +# Bail if still unreachable +if ! mysqladmin --socket="$SOCKET" ping "${MYSQL_ROOT_FLAGS[@]}" >/dev/null 2>&1; then + exit 0 +fi + +# Create database and user idempotently (MySQL 8+ supports IF NOT EXISTS for users) +DB=${MYSQL_DATABASE:-singularity} +USER=${MYSQL_USER:-singularity} +PASS=${MYSQL_PASSWORD:-singularity} + +mysql --socket="$SOCKET" "${MYSQL_ROOT_FLAGS[@]}" </dev/null 2>&1; then + echo "MySQL already running" + exit 0 +fi + +echo "Starting MySQL server..." +mysql.server start + +# MySQL uses /tmp/mysql.sock by default when started with mysql.server +SOCKET="/tmp/mysql.sock" + +# Wait for MySQL to be ready +for i in {1..60}; do + if mysqladmin --socket="$SOCKET" --user=root ping >/dev/null 2>&1; then + echo "MySQL server is ready" + exit 0 + fi + sleep 1 +done + +echo "MySQL server failed to start" +exit 1 \ No newline at end of file From 41cbf8608470fef97a1b66aee0f4c0a1f5a8e37f Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 15 Oct 2025 16:45:34 +0200 Subject: [PATCH 02/13] add devcontainer-based ci tests --- .github/workflows/test-devcontainer.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/test-devcontainer.yaml diff --git a/.github/workflows/test-devcontainer.yaml b/.github/workflows/test-devcontainer.yaml new file mode 100644 index 00000000..2ac5ef7f --- /dev/null +++ b/.github/workflows/test-devcontainer.yaml @@ -0,0 +1,18 @@ +name: Test with Devcontainer +on: + push: + branches: [ "local/devcontainer" ] + pull_request: + branches: [ "main" ] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run tests in devcontainer + uses: devcontainers/ci@v0.3 + with: + runCmd: make test \ No newline at end of file From 3ca919abc5bc2887efb99d485458be0962b89e33 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 15 Oct 2025 16:45:34 +0200 Subject: [PATCH 03/13] configure podman-based devcontainer with ci workflow - workarounds for volume relabeling issues - podman-specific configurations - devcontainer re-engineering for podman compatibility - missing scripts and permission fixes --- .devcontainer/Dockerfile | 36 +++++++++++++++ .devcontainer/devcontainer.json | 32 +++++++------ .devcontainer/init-mysql.sh | 13 ++---- .devcontainer/post-create.sh | 9 ++++ .devcontainer/start-mysql.sh | 56 ++++++++++++++++++----- .devcontainer/start-postgres.sh | 24 ++++++++++ .github/workflows/devcontainer-podman.yml | 28 ++++++++++++ .github/workflows/test-devcontainer.yaml | 18 ++------ 8 files changed, 167 insertions(+), 49 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100755 .devcontainer/post-create.sh create mode 100755 .devcontainer/start-postgres.sh create mode 100644 .github/workflows/devcontainer-podman.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..2fccf82f --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,36 @@ +ARG GO_VERSION=1.24 +ARG DEBIAN_CODENAME=trixie + +FROM mcr.microsoft.com/devcontainers/go:${GO_VERSION}-${DEBIAN_CODENAME} + +ARG PG_MAJOR=16 +ARG MARIADB_MAJOR=12.0.2 + +USER root + +# Add PostgreSQL and MariaDB official repositories (detect codename from /etc/os-release) +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + wget ca-certificates gnupg curl \ + && wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg \ + && echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo $VERSION_CODENAME)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && curl -LsSO https://r.mariadb.com/downloads/mariadb_repo_setup \ + && chmod +x mariadb_repo_setup \ + && ./mariadb_repo_setup --mariadb-server-version="mariadb-${MARIADB_MAJOR}" --skip-maxscale --skip-tools \ + && rm -f mariadb_repo_setup \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + postgresql-${PG_MAJOR} postgresql-client-${PG_MAJOR} \ + mariadb-server mariadb-client \ + && rm -rf /var/lib/apt/lists/* + +# Prepare user-owned data dirs for rootless startup +RUN mkdir -p /home/vscode/.local/share/pg/pgdata \ + && mkdir -p /home/vscode/.local/share/mysql \ + && chown -R vscode:vscode /home/vscode/.local/share + +# Set environment variables based on build args +ENV PG_BIN_DIR=/usr/lib/postgresql/${PG_MAJOR}/bin + +USER vscode + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 64b5b784..f0e2ff23 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,14 @@ { "name": "singularity", - "image": "mcr.microsoft.com/devcontainers/go:1.24", + "build": { + "dockerfile": "Dockerfile", + "args": { + "GO_VERSION": "1.24", + "DEBIAN_CODENAME": "trixie", + "PG_MAJOR": "16", + "MARIADB_MAJOR": "12.0.2" + } + }, "customizations": { "vscode": { "extensions": [ @@ -15,27 +23,21 @@ } } }, - "features": { - "ghcr.io/devcontainers/features/go:1": { - "version": "1.24" - }, - "ghcr.io/itsmechlark/features/postgresql:1": { - "version": "16" - }, - "ghcr.io/devcontainers-extra/features/mysql-homebrew:1": {} - }, "forwardPorts": [], - "postCreateCommand": "go mod download", - "postStartCommand": "/bin/bash -lc 'echo \"local all all trust\nhost all all 127.0.0.1/32 trust\nhost all all ::1/128 trust\" | sudo tee /etc/postgresql/16/main/pg_hba.conf > /dev/null && sudo service postgresql restart && /workspace/.devcontainer/start-mysql.sh && /workspace/.devcontainer/init-mysql.sh'", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/singularity,type=bind,consistency=cached,relabel=private", + "workspaceFolder": "/workspaces/singularity", + "postCreateCommand": "/bin/bash -lc '${containerWorkspaceFolder}/.devcontainer/post-create.sh'", + "postStartCommand": "/bin/bash -lc '${containerWorkspaceFolder}/.devcontainer/start-postgres.sh && ${containerWorkspaceFolder}/.devcontainer/start-mysql.sh && ${containerWorkspaceFolder}/.devcontainer/init-mysql.sh'", "remoteUser": "vscode", - "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached,relabel=private", - "workspaceFolder": "/workspace", "containerEnv": { "GOPATH": "/home/vscode/go", "MYSQL_DATABASE": "singularity", "MYSQL_USER": "singularity", "MYSQL_PASSWORD": "singularity", - "MYSQL_SOCKET": "/tmp/mysql.sock" + "MYSQL_SOCKET": "/home/vscode/.local/share/mysql/mysql.sock", + "PGDATA": "/home/vscode/.local/share/pg/pgdata", + "PGPORT": "55432", + "PGSOCK_DIR": "/home/vscode/.local/share/pg" }, "runArgs": ["--userns=keep-id"], "containerUser": "vscode" diff --git a/.devcontainer/init-mysql.sh b/.devcontainer/init-mysql.sh index 0f0a7265..94da58ca 100755 --- a/.devcontainer/init-mysql.sh +++ b/.devcontainer/init-mysql.sh @@ -6,14 +6,10 @@ if ! command -v mysql >/dev/null 2>&1; then exit 0 fi -# Use same socket as mysql.server (homebrew default) -SOCKET="${MYSQL_SOCKET:-/tmp/mysql.sock}" +# Resolve socket path; default to user-owned socket +SOCKET="${MYSQL_SOCKET:-${HOME}/.local/share/mysql/mysql.sock}" -# One-time init guard -MARKER="/workspace/.devcontainer/.mysql-initialized" -if [ -f "$MARKER" ]; then - exit 0 -fi +## Removed one-time init guard; operations below are idempotent # Determine root auth flags MYSQL_ROOT_FLAGS=("-uroot") @@ -48,7 +44,6 @@ GRANT ALL PRIVILEGES ON \`${DB}\`.* TO '${USER}'@'%'; FLUSH PRIVILEGES; SQL -# mark initialized -touch "$MARKER" +## No marker file; script can run safely on every start diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh new file mode 100755 index 00000000..2ac5dda3 --- /dev/null +++ b/.devcontainer/post-create.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Initialize Go modules +go mod download + +chmod +x .devcontainer/*.sh || true + + diff --git a/.devcontainer/start-mysql.sh b/.devcontainer/start-mysql.sh index cf6092f9..673f2d27 100755 --- a/.devcontainer/start-mysql.sh +++ b/.devcontainer/start-mysql.sh @@ -1,24 +1,53 @@ #!/usr/bin/env bash set -euo pipefail -# Start MySQL using homebrew's mysql.server command -# This is provided by the ghcr.io/devcontainers-extra/features/mysql-homebrew feature +MYSQLD_BIN="$(command -v mariadbd || true)" +if [ -z "$MYSQLD_BIN" ]; then + MYSQLD_BIN="$(command -v mysqld || true)" +fi +if [ -z "$MYSQLD_BIN" ]; then + echo "Could not find MariaDB server binary (mariadbd or mysqld)" + exit 1 +fi + +# Start MariaDB as an unprivileged user using a user-owned data directory + +MYSQL_BASE="${HOME}/.local/share/mysql" +DATA_DIR="${MYSQL_BASE}/data" +SOCKET="${MYSQL_SOCKET:-${MYSQL_BASE}/mysql.sock}" +PID_FILE="${MYSQL_BASE}/mysql.pid" +PORT="${MYSQL_PORT:-3306}" +LOG_FILE="${MYSQL_BASE}/mysql.err" + +mkdir -p "${DATA_DIR}" "${MYSQL_BASE}" -# Check if already running -if mysql.server status >/dev/null 2>&1; then +# Initialize data dir if missing +if [ ! -d "${DATA_DIR}/mysql" ]; then + echo "Initializing MariaDB data directory" + mariadb-install-db --datadir="${DATA_DIR}" --auth-root-authentication-method=normal --skip-test-db >/dev/null +fi + +# If already running, exit +if [ -S "${SOCKET}" ] && mysqladmin --socket="${SOCKET}" ping >/dev/null 2>&1; then echo "MySQL already running" exit 0 fi -echo "Starting MySQL server..." -mysql.server start +echo "Starting MySQL server" +touch "${LOG_FILE}" +nohup "$MYSQLD_BIN" \ + --datadir="${DATA_DIR}" \ + --socket="${SOCKET}" \ + --pid-file="${PID_FILE}" \ + --bind-address=127.0.0.1 \ + --port="${PORT}" \ + --skip-name-resolve \ + --log-error="${LOG_FILE}" \ + >/dev/null 2>&1 & -# MySQL uses /tmp/mysql.sock by default when started with mysql.server -SOCKET="/tmp/mysql.sock" - -# Wait for MySQL to be ready +# Wait for MySQL to be ready (log-based + socket existence) for i in {1..60}; do - if mysqladmin --socket="$SOCKET" --user=root ping >/dev/null 2>&1; then + if [ -S "${SOCKET}" ] && grep -q "ready for connections" "${LOG_FILE}" >/dev/null 2>&1; then echo "MySQL server is ready" exit 0 fi @@ -26,4 +55,9 @@ for i in {1..60}; do done echo "MySQL server failed to start" +if [ -f "${LOG_FILE}" ]; then + echo "--- Begin MariaDB error log ---" + tail -n 200 "${LOG_FILE}" || true + echo "--- End MariaDB error log ---" +fi exit 1 \ No newline at end of file diff --git a/.devcontainer/start-postgres.sh b/.devcontainer/start-postgres.sh new file mode 100755 index 00000000..af02d3e6 --- /dev/null +++ b/.devcontainer/start-postgres.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +PGDATA_DIR="${PGDATA:-/home/vscode/.local/share/pg/pgdata}" +LOG_FILE="${PGDATA_DIR}/postgres.log" +PGSOCK_DIR="${PGSOCK_DIR:-/home/vscode/.local/share/pg}" +PGPORT="${PGPORT:-55432}" +PG_BIN_DIR="${PG_BIN_DIR:-/usr/lib/postgresql/16/bin}" + +mkdir -p "$PGDATA_DIR" "$PGSOCK_DIR" + +if [ ! -f "$PGDATA_DIR/PG_VERSION" ]; then + "${PG_BIN_DIR}/initdb" -D "$PGDATA_DIR" --auth trust --auth-local trust --auth-host trust + echo "listen_addresses='localhost'" >> "$PGDATA_DIR/postgresql.conf" + { + echo 'host all all 127.0.0.1/32 trust' + echo 'host all all ::1/128 trust' + echo 'local all all trust' + } >> "$PGDATA_DIR/pg_hba.conf" +fi + +"${PG_BIN_DIR}/pg_ctl" -D "$PGDATA_DIR" -l "$LOG_FILE" -w -o "-p ${PGPORT} -k ${PGSOCK_DIR}" start + + diff --git a/.github/workflows/devcontainer-podman.yml b/.github/workflows/devcontainer-podman.yml new file mode 100644 index 00000000..3d80c1fd --- /dev/null +++ b/.github/workflows/devcontainer-podman.yml @@ -0,0 +1,28 @@ +name: CI (Podman Devcontainer) + +on: + push: + branches: [ local/devcontainer ] + pull_request: + branches: [ local/devcontainer ] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Podman as Docker + uses: parkan/github-actions/setup-podman-docker@v1 + with: + disable-docker: true + cache-storage: false # Disabled until cache issues resolved + + - name: Run Devcontainer Tests + uses: parkan/github-actions/devcontainer-test@v1 + with: + workspace-folder: . + container-runtime: podman + container-id-label: ci=podman + test-command: 'cd /workspaces/singularity && make test' \ No newline at end of file diff --git a/.github/workflows/test-devcontainer.yaml b/.github/workflows/test-devcontainer.yaml index 2ac5ef7f..867924f2 100644 --- a/.github/workflows/test-devcontainer.yaml +++ b/.github/workflows/test-devcontainer.yaml @@ -1,18 +1,8 @@ -name: Test with Devcontainer +name: disabled (replaced by Podman devcontainer workflow) on: - push: - branches: [ "local/devcontainer" ] - pull_request: - branches: [ "main" ] - + workflow_call: jobs: - test: + skip: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Run tests in devcontainer - uses: devcontainers/ci@v0.3 - with: - runCmd: make test \ No newline at end of file + - run: echo "This workflow is disabled. See devcontainer-podman.yml." \ No newline at end of file From 5e510fa3d336026c6f2f3bbc2daafe082fab92a3 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 15 Oct 2025 16:45:34 +0200 Subject: [PATCH 04/13] fix mysql configuration and database connectivity - mysql user and permission fixes - database role creation - test database setup --- .devcontainer/devcontainer.json | 2 +- .devcontainer/init-mysql.sh | 22 +++++--- .devcontainer/post-create.sh | 46 +++++++++++++++++ .devcontainer/start-mysql.sh | 29 +++-------- .devcontainer/start-postgres.sh | 17 +++--- util/testutil/testutils.go | 91 ++++++++++++++++++++++++++------- 6 files changed, 147 insertions(+), 60 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f0e2ff23..f2c092ac 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -27,7 +27,7 @@ "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/singularity,type=bind,consistency=cached,relabel=private", "workspaceFolder": "/workspaces/singularity", "postCreateCommand": "/bin/bash -lc '${containerWorkspaceFolder}/.devcontainer/post-create.sh'", - "postStartCommand": "/bin/bash -lc '${containerWorkspaceFolder}/.devcontainer/start-postgres.sh && ${containerWorkspaceFolder}/.devcontainer/start-mysql.sh && ${containerWorkspaceFolder}/.devcontainer/init-mysql.sh'", + "postStartCommand": "/bin/bash -lc '${containerWorkspaceFolder}/.devcontainer/start-postgres.sh && ${containerWorkspaceFolder}/.devcontainer/start-mysql.sh'", "remoteUser": "vscode", "containerEnv": { "GOPATH": "/home/vscode/go", diff --git a/.devcontainer/init-mysql.sh b/.devcontainer/init-mysql.sh index 94da58ca..b9e3f1af 100755 --- a/.devcontainer/init-mysql.sh +++ b/.devcontainer/init-mysql.sh @@ -1,10 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -# If mysql client isn't present, exit quietly -if ! command -v mysql >/dev/null 2>&1; then - exit 0 -fi +# MariaDB client is required for init # Resolve socket path; default to user-owned socket SOCKET="${MYSQL_SOCKET:-${HOME}/.local/share/mysql/mysql.sock}" @@ -18,16 +15,19 @@ if [ -n "${MYSQL_ROOT_PASSWORD:-}" ]; then fi # Wait for server readiness (best effort) +echo "Waiting for MySQL server at socket: $SOCKET" for i in {1..60}; do - if mysqladmin --socket="$SOCKET" ping "${MYSQL_ROOT_FLAGS[@]}" >/dev/null 2>&1; then + if mariadb-admin --socket="$SOCKET" ping "${MYSQL_ROOT_FLAGS[@]}" >/dev/null 2>&1; then + echo "MySQL server is ready for init" break fi sleep 1 done # Bail if still unreachable -if ! mysqladmin --socket="$SOCKET" ping "${MYSQL_ROOT_FLAGS[@]}" >/dev/null 2>&1; then - exit 0 +if ! mariadb-admin --socket="$SOCKET" ping "${MYSQL_ROOT_FLAGS[@]}" >/dev/null 2>&1; then + echo "MySQL server not reachable, init failed" + exit 1 fi # Create database and user idempotently (MySQL 8+ supports IF NOT EXISTS for users) @@ -35,15 +35,21 @@ DB=${MYSQL_DATABASE:-singularity} USER=${MYSQL_USER:-singularity} PASS=${MYSQL_PASSWORD:-singularity} -mysql --socket="$SOCKET" "${MYSQL_ROOT_FLAGS[@]}" <> /home/vscode/.local/share/pg/pgdata/postgresql.conf + { + echo 'host all all 127.0.0.1/32 trust' + echo 'host all all ::1/128 trust' + echo 'local all all trust' + } >> /home/vscode/.local/share/pg/pgdata/pg_hba.conf +fi + +# Initialize MariaDB +if [ ! -d "/home/vscode/.local/share/mysql/data/mysql" ]; then + echo "Initializing MariaDB..." + mariadb-install-db --datadir=/home/vscode/.local/share/mysql/data --auth-root-authentication-method=normal --skip-test-db >/dev/null +fi + +# Start both servers +echo "Starting database servers..." +.devcontainer/start-postgres.sh +.devcontainer/start-mysql.sh + +# Create users (databases will be created during testing as needed) +echo "Creating database users..." + +# Postgres setup +psql -h localhost -p 55432 -d postgres -c "CREATE USER singularity WITH SUPERUSER CREATEDB CREATEROLE LOGIN;" + +# MySQL setup +mariadb --socket=/home/vscode/.local/share/mysql/mysql.sock -uroot </dev/null -fi - -# If already running, exit -if [ -S "${SOCKET}" ] && mysqladmin --socket="${SOCKET}" ping >/dev/null 2>&1; then +# Check if already running +if [ -S "${SOCKET}" ] && mariadb-admin --socket="${SOCKET}" ping >/dev/null 2>&1; then echo "MySQL already running" exit 0 fi +# Start MariaDB server echo "Starting MySQL server" touch "${LOG_FILE}" -nohup "$MYSQLD_BIN" \ +nohup mariadbd \ --datadir="${DATA_DIR}" \ --socket="${SOCKET}" \ --pid-file="${PID_FILE}" \ @@ -45,7 +28,7 @@ nohup "$MYSQLD_BIN" \ --log-error="${LOG_FILE}" \ >/dev/null 2>&1 & -# Wait for MySQL to be ready (log-based + socket existence) +# Wait for MySQL to be ready for i in {1..60}; do if [ -S "${SOCKET}" ] && grep -q "ready for connections" "${LOG_FILE}" >/dev/null 2>&1; then echo "MySQL server is ready" diff --git a/.devcontainer/start-postgres.sh b/.devcontainer/start-postgres.sh index af02d3e6..b90cb1a8 100755 --- a/.devcontainer/start-postgres.sh +++ b/.devcontainer/start-postgres.sh @@ -1,24 +1,21 @@ #!/usr/bin/env bash set -euo pipefail +# Postgres server configuration PGDATA_DIR="${PGDATA:-/home/vscode/.local/share/pg/pgdata}" LOG_FILE="${PGDATA_DIR}/postgres.log" PGSOCK_DIR="${PGSOCK_DIR:-/home/vscode/.local/share/pg}" PGPORT="${PGPORT:-55432}" PG_BIN_DIR="${PG_BIN_DIR:-/usr/lib/postgresql/16/bin}" -mkdir -p "$PGDATA_DIR" "$PGSOCK_DIR" - -if [ ! -f "$PGDATA_DIR/PG_VERSION" ]; then - "${PG_BIN_DIR}/initdb" -D "$PGDATA_DIR" --auth trust --auth-local trust --auth-host trust - echo "listen_addresses='localhost'" >> "$PGDATA_DIR/postgresql.conf" - { - echo 'host all all 127.0.0.1/32 trust' - echo 'host all all ::1/128 trust' - echo 'local all all trust' - } >> "$PGDATA_DIR/pg_hba.conf" +# Check if already running +if "${PG_BIN_DIR}/pg_ctl" -D "$PGDATA_DIR" status >/dev/null 2>&1; then + echo "Postgres already running" + exit 0 fi +# Start Postgres server +echo "Starting Postgres server" "${PG_BIN_DIR}/pg_ctl" -D "$PGDATA_DIR" -l "$LOG_FILE" -w -o "-p ${PGPORT} -k ${PGSOCK_DIR}" start diff --git a/util/testutil/testutils.go b/util/testutil/testutils.go index 064fdbe5..66d026c9 100644 --- a/util/testutil/testutils.go +++ b/util/testutil/testutils.go @@ -5,8 +5,8 @@ import ( "crypto/rand" "io" rand2 "math/rand" - "net" "os" + "os/exec" "strings" "testing" "time" @@ -14,6 +14,7 @@ import ( "github.com/cockroachdb/errors" "github.com/data-preservation-programs/singularity/database" "github.com/data-preservation-programs/singularity/model" + "github.com/google/uuid" "github.com/ipfs/boxo/util" "github.com/ipfs/go-cid" "github.com/stretchr/testify/require" @@ -49,6 +50,11 @@ func RandomLetterString(length int) string { return string(b) } +// GenerateUniqueName creates a unique name for testing by combining a prefix with a UUID suffix +func GenerateUniqueName(prefix string) string { + return prefix + "-" + strings.ReplaceAll(uuid.New().String(), "-", "") +} + func GetFileTimestamp(t *testing.T, path string) int64 { t.Helper() info, err := os.Stat(path) @@ -75,33 +81,82 @@ func getTestDB(t *testing.T, dialect string) (db *gorm.DB, closer io.Closer, con require.NoError(t, err) return } - dbName := RandomLetterString(6) - var opError *net.OpError + // Use UUID for database names to ensure uniqueness and avoid MySQL's 64-character limit + // Remove hyphens to make it a valid database identifier + dbName := "test_" + strings.ReplaceAll(uuid.New().String(), "-", "") switch dialect { case "mysql": - connStr = "mysql://singularity:singularity@tcp(localhost:3306)/singularity?parseTime=true" + socket := os.Getenv("MYSQL_SOCKET") + connStr = "mysql://singularity:singularity@unix(" + socket + ")/mysql?parseTime=true" case "postgres": - connStr = "postgres://singularity:singularity@localhost:5432/singularity?sslmode=disable" + pgPort := os.Getenv("PGPORT") + connStr = "postgres://singularity@localhost:" + pgPort + "/postgres?sslmode=disable" default: require.Fail(t, "Unsupported dialect: "+dialect) } - var db1 *gorm.DB - var closer1 io.Closer - db1, closer1, err = database.OpenWithLogger(connStr) - if errors.As(err, &opError) { - return + // Skip initial connection test - databases will be created during testing + // Create database using shell commands to avoid driver transaction issues + switch dialect { + case "postgres": + // Use createdb command for PostgreSQL + cmd := exec.Command("createdb", "-h", "localhost", "-p", os.Getenv("PGPORT"), "-U", "singularity", dbName) + output, err := cmd.CombinedOutput() + if err != nil { + t.Logf("Failed to create PostgreSQL database %s: %v, output: %s", dbName, err, string(output)) + return nil, nil, "" + } + t.Logf("Created PostgreSQL database %s", dbName) + case "mysql": + // Use mysql command for MySQL + socket := os.Getenv("MYSQL_SOCKET") + cmd := exec.Command("mariadb", "--socket="+socket, "-usingularity", "-psingularity", "-e", "CREATE DATABASE "+dbName) + output, err := cmd.CombinedOutput() + if err != nil { + t.Logf("Failed to create MySQL database %s: %v, output: %s", dbName, err, string(output)) + return nil, nil, "" + } + t.Logf("Created MySQL database %s", dbName) + default: + t.Logf("Unsupported dialect for shell database creation: %s", dialect) + return nil, nil, "" + } + // Replace database name in connection string + if strings.Contains(connStr, "postgres?") { + connStr = strings.ReplaceAll(connStr, "postgres?", dbName+"?") + } else if strings.Contains(connStr, "mysql?") { + connStr = strings.ReplaceAll(connStr, "mysql?", dbName+"?") } - require.NoError(t, err) - err = db1.Exec("CREATE DATABASE " + dbName + "").Error - require.NoError(t, err) - connStr = strings.ReplaceAll(connStr, "singularity?", dbName+"?") var closer2 io.Closer db, closer2, err = database.OpenWithLogger(connStr) - require.NoError(t, err) + if err != nil { + t.Logf("Failed to connect to test database %s: %v", dbName, err) + // Cleanup using shell commands + switch dialect { + case "postgres": + cmd := exec.Command("dropdb", "-h", "localhost", "-p", os.Getenv("PGPORT"), "-U", "singularity", dbName) + cmd.Run() // Ignore errors during cleanup + case "mysql": + socket := os.Getenv("MYSQL_SOCKET") + cmd := exec.Command("mariadb", "--socket="+socket, "-usingularity", "-psingularity", "-e", "DROP DATABASE "+dbName) + cmd.Run() // Ignore errors during cleanup + } + return nil, nil, "" + } closer = CloserFunc(func() error { - require.NoError(t, closer2.Close()) - require.NoError(t, db1.Exec("DROP DATABASE "+dbName+"").Error) - return closer1.Close() + if closer2 != nil { + _ = closer2.Close() + } + // Cleanup using shell commands + switch dialect { + case "postgres": + cmd := exec.Command("dropdb", "-h", "localhost", "-p", os.Getenv("PGPORT"), "-U", "singularity", dbName) + cmd.Run() // Ignore errors during cleanup + case "mysql": + socket := os.Getenv("MYSQL_SOCKET") + cmd := exec.Command("mariadb", "--socket="+socket, "-usingularity", "-psingularity", "-e", "DROP DATABASE "+dbName) + cmd.Run() // Ignore errors during cleanup + } + return nil }) return } From 1c4479f0b71286aed75be7803fcafb80e926979e Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 15 Oct 2025 16:45:34 +0200 Subject: [PATCH 05/13] fix database test issues and race conditions - job selection logic fix - suppress missing table warnings in tests - disable dag generation during duplicate piece test - resolve record changed since last read errors --- cmd/functional_test.go | 2 +- database/util.go | 18 +++++++++++++++++- service/datasetworker/find.go | 10 +++++----- util/testutil/testutils.go | 30 ++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/cmd/functional_test.go b/cmd/functional_test.go index 14009ea6..d663950e 100644 --- a/cmd/functional_test.go +++ b/cmd/functional_test.go @@ -513,7 +513,7 @@ func TestNoDuplicatedOutput(t *testing.T) { // run the dataset worker. If multiple workers try to work on the same // job, then this will return fail because a previous worker will have // removed files. - _, _, err = runner.Run(ctx, "singularity run dataset-worker --exit-on-complete=true --exit-on-error=true --concurrency=8") + _, _, err = runner.Run(ctx, "singularity run dataset-worker --exit-on-complete=true --exit-on-error=true --concurrency=8 --enable-dag=false") require.NoError(t, err) // Check output to make sure is has some CAR files diff --git a/database/util.go b/database/util.go index 1df11f31..88030bc3 100644 --- a/database/util.go +++ b/database/util.go @@ -66,7 +66,13 @@ func (d *databaseLogger) Trace(ctx context.Context, begin time.Time, fc func() ( sql = "[SLOW!] " + sql } if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) && !strings.Contains(err.Error(), sqlSerializationFailure) { - lvl = logging.LevelError + // Demote noisy missing-table errors during test setup/teardown + emsg := err.Error() + if strings.Contains(emsg, "no such table") || strings.Contains(emsg, "does not exist") || strings.Contains(emsg, "doesn't exist") { + lvl = logging.LevelDebug + } else { + lvl = logging.LevelError + } } // Uncomment for logging everything in testing @@ -95,3 +101,13 @@ func OpenFromCLI(c *cli.Context) (*gorm.DB, io.Closer, error) { connString := c.String("database-connection-string") return OpenWithLogger(connString) } + +func retryOn(err error) bool { + emsg := err.Error() + return strings.Contains(emsg, sqlSerializationFailure) || + strings.Contains(emsg, "database is locked") || + strings.Contains(emsg, "database table is locked") || + // MySQL/InnoDB serialization conflict + strings.Contains(emsg, "Record has changed since last read") || + strings.Contains(emsg, "Error 1020 (HY000)") +} diff --git a/service/datasetworker/find.go b/service/datasetworker/find.go index b8077a89..367c6915 100644 --- a/service/datasetworker/find.go +++ b/service/datasetworker/find.go @@ -30,11 +30,11 @@ func (w *Thread) findJob(ctx context.Context, typesOrdered []model.JobType) (*mo } var job model.Job for _, jobType := range typesOrdered { - err := database.DoRetry(ctx, func() error { - return db.Transaction(func(db *gorm.DB) error { - err := db.Preload("Attachment.Preparation.OutputStorages").Preload("Attachment.Storage"). - Where("type = ? AND state = ? OR (state = ? AND worker_id is null)", jobType, model.Ready, model.Processing). - First(&job).Error + err := database.DoRetry(ctx, func() error { + return db.Transaction(func(db *gorm.DB) error { + err := db.Preload("Attachment.Preparation.OutputStorages").Preload("Attachment.Storage"). + Where("type = ? AND (state = ? OR (state = ? AND worker_id IS NULL))", jobType, model.Ready, model.Processing). + First(&job).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { job.ID = 0 diff --git a/util/testutil/testutils.go b/util/testutil/testutils.go index 66d026c9..23312fa1 100644 --- a/util/testutil/testutils.go +++ b/util/testutil/testutils.go @@ -202,6 +202,36 @@ func doOne(t *testing.T, backend string, testFunc func(ctx context.Context, t *t err := model.AutoMigrate(db) require.NoError(t, err) + // Clear any existing data from tables with unique constraints + tables := []string{ + "output_attachments", + "source_attachments", + "storages", + "wallets", + "deal_schedules", + "preparations", + } + + // Get DB type from connection string + isPostgres := strings.HasPrefix(connStr, "postgres:") + for _, table := range tables { + var err error + if isPostgres { + err = db.Exec("TRUNCATE TABLE " + table + " CASCADE").Error + } else { + err = db.Exec("DELETE FROM " + table).Error + } + if err != nil { + emsg := err.Error() + // Suppress noisy logs when tables don't exist yet across backends + if strings.Contains(emsg, "no such table") || strings.Contains(emsg, "does not exist") || strings.Contains(emsg, "doesn't exist") { + continue + } + t.Logf("Warning: Failed to clear table %s: %v", table, err) + // Don't fail the test for other errors either + } + } + t.Run(backend, func(t *testing.T) { testFunc(ctx, t, db) }) From f1182fef9c0735436bd0f82032f4f89fe76fdd0b Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 15 Oct 2025 16:45:34 +0200 Subject: [PATCH 06/13] optimize devcontainer ci workflow - use v2 action - reorder steps for fast fail - remove stray files --- .github/workflows/devcontainer-podman.yml | 80 +++++++++++++++++++++-- .github/workflows/test-devcontainer.yaml | 8 --- 2 files changed, 73 insertions(+), 15 deletions(-) delete mode 100644 .github/workflows/test-devcontainer.yaml diff --git a/.github/workflows/devcontainer-podman.yml b/.github/workflows/devcontainer-podman.yml index 3d80c1fd..84934b1e 100644 --- a/.github/workflows/devcontainer-podman.yml +++ b/.github/workflows/devcontainer-podman.yml @@ -2,27 +2,93 @@ name: CI (Podman Devcontainer) on: push: - branches: [ local/devcontainer ] + branches: [ main, develop ] pull_request: - branches: [ local/devcontainer ] + branches: [ main, develop ] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true jobs: - test: + devcontainer-checks: + name: Devcontainer CI runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Podman as Docker - uses: parkan/github-actions/setup-podman-docker@v1 + uses: parkan/github-actions/setup-podman-docker@v2 with: disable-docker: true cache-storage: false # Disabled until cache issues resolved - - name: Run Devcontainer Tests - uses: parkan/github-actions/devcontainer-test@v1 + - name: Build devcontainer + id: build + uses: parkan/github-actions/devcontainer-build@v2 with: workspace-folder: . container-runtime: podman container-id-label: ci=podman - test-command: 'cd /workspaces/singularity && make test' \ No newline at end of file + + # Fast surface checks and codegen first + - name: Generate swagger code + uses: parkan/github-actions/devcontainer-exec@v2 + with: + container-id: ${{ steps.build.outputs.container-id }} + command: 'cd /workspaces/singularity && mkdir -p client/swagger/client && go install github.com/go-swagger/go-swagger/cmd/swagger@v0.30.5 && go generate ./client/swagger/...' + container-runtime: podman + + - name: Check formatting + uses: parkan/github-actions/devcontainer-exec@v2 + with: + container-id: ${{ steps.build.outputs.container-id }} + command: 'cd /workspaces/singularity && gofmt -l .' + container-runtime: podman + + - name: Run go vet + uses: parkan/github-actions/devcontainer-exec@v2 + with: + container-id: ${{ steps.build.outputs.container-id }} + command: 'cd /workspaces/singularity && go vet ./...' + container-runtime: podman + + - name: Run staticcheck + uses: parkan/github-actions/devcontainer-exec@v2 + with: + container-id: ${{ steps.build.outputs.container-id }} + command: 'cd /workspaces/singularity && staticcheck ./...' + container-runtime: podman + + - name: Build binary + uses: parkan/github-actions/devcontainer-exec@v2 + with: + container-id: ${{ steps.build.outputs.container-id }} + command: 'cd /workspaces/singularity && go build -o singularity .' + container-runtime: podman + + - name: Run tests + uses: parkan/github-actions/devcontainer-exec@v2 + with: + container-id: ${{ steps.build.outputs.container-id }} + command: 'cd /workspaces/singularity && go test -v ./...' + container-runtime: podman + + - name: Run integration tests + uses: parkan/github-actions/devcontainer-exec@v2 + with: + container-id: ${{ steps.build.outputs.container-id }} + command: 'cd /workspaces/singularity && SINGULARITY_TEST_INTEGRATION=true go test -v -timeout 20m -run "Integration" ./cmd/...' + container-runtime: podman + + - name: Cleanup + if: always() + uses: parkan/github-actions/devcontainer-cleanup@v2 + with: + container-id: ${{ steps.build.outputs.container-id }} + container-runtime: podman diff --git a/.github/workflows/test-devcontainer.yaml b/.github/workflows/test-devcontainer.yaml deleted file mode 100644 index 867924f2..00000000 --- a/.github/workflows/test-devcontainer.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: disabled (replaced by Podman devcontainer workflow) -on: - workflow_call: -jobs: - skip: - runs-on: ubuntu-latest - steps: - - run: echo "This workflow is disabled. See devcontainer-podman.yml." \ No newline at end of file From f1def13315dc319ca8b2dfd427460ca08a72c20e Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 15 Oct 2025 16:45:34 +0200 Subject: [PATCH 07/13] remove heavyweight unified workflows --- .github/workflows/go-check-config.json | 3 --- .github/workflows/go-check.yml | 18 ------------------ .github/workflows/go-test-config.json | 3 --- .github/workflows/go-test.yml | 18 ------------------ 4 files changed, 42 deletions(-) delete mode 100644 .github/workflows/go-check-config.json delete mode 100644 .github/workflows/go-check.yml delete mode 100644 .github/workflows/go-test-config.json delete mode 100644 .github/workflows/go-test.yml diff --git a/.github/workflows/go-check-config.json b/.github/workflows/go-check-config.json deleted file mode 100644 index 4b37308d..00000000 --- a/.github/workflows/go-check-config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "gogenerate": true -} diff --git a/.github/workflows/go-check.yml b/.github/workflows/go-check.yml deleted file mode 100644 index 6972415d..00000000 --- a/.github/workflows/go-check.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Go Checks - -on: - pull_request: - push: - branches: ["main"] - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} - cancel-in-progress: true - -jobs: - go-check: - uses: ipdxco/unified-github-workflows/.github/workflows/go-check.yml@v1.0.22 diff --git a/.github/workflows/go-test-config.json b/.github/workflows/go-test-config.json deleted file mode 100644 index 209dca21..00000000 --- a/.github/workflows/go-test-config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "skip32bit": true -} diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml deleted file mode 100644 index cb43a3ae..00000000 --- a/.github/workflows/go-test.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Go Test - -on: - pull_request: - push: - branches: ["main"] - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} - cancel-in-progress: true - -jobs: - go-test: - uses: ipdxco/unified-github-workflows/.github/workflows/go-test.yml@v1.0 From 7dd9ce7d010d5b17596350c39bc2043bac57a1dc Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Thu, 16 Oct 2025 11:40:32 +0000 Subject: [PATCH 08/13] remove 386 builds --- .goreleaser.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index a67c1e00..74382123 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -11,7 +11,6 @@ builds: goarch: - amd64 - arm64 - - 386 mod_timestamp: '{{.CommitTimestamp}}' archives: From a44c40c03a10a00c6f96f45c739a444617fc0358 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Thu, 16 Oct 2025 11:43:24 +0000 Subject: [PATCH 09/13] enable images for testing --- .github/workflows/container-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container-publish.yml b/.github/workflows/container-publish.yml index 8a35a737..e49bde56 100644 --- a/.github/workflows/container-publish.yml +++ b/.github/workflows/container-publish.yml @@ -4,6 +4,7 @@ on: push: branches: - 'main' + - 'develop' tags: - 'v*' workflow_run: From ebb289155c65d55d26b379756d68717ec638ddbf Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Thu, 16 Oct 2025 11:54:20 +0000 Subject: [PATCH 10/13] harmonize Docker golang version; can't sync this sadly --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1fbcc621..74456363 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23.6-bullseye AS builder +FROM golang:1.24.0-bullseye AS builder WORKDIR /app COPY go.* ./ RUN go mod download From c1f1b23c212373fb51cdd9491b354e1b1dede341 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 22 Oct 2025 12:28:27 +0200 Subject: [PATCH 11/13] fixup on rebase divergence --- database/util.go | 5 ----- util/testutil/testutils.go | 1 - 2 files changed, 6 deletions(-) diff --git a/database/util.go b/database/util.go index 88030bc3..062b23b6 100644 --- a/database/util.go +++ b/database/util.go @@ -24,11 +24,6 @@ var ( ErrDatabaseNotSupported = errors.New("database not supported") ) -func retryOn(err error) bool { - emsg := err.Error() - return strings.Contains(emsg, sqlSerializationFailure) || strings.Contains(emsg, "database is locked") || strings.Contains(emsg, "database table is locked") -} - func DoRetry(ctx context.Context, f func() error) error { return retry.Do(f, retry.RetryIf(retryOn), retry.LastErrorOnly(true), retry.Context(ctx)) } diff --git a/util/testutil/testutils.go b/util/testutil/testutils.go index 23312fa1..7e544846 100644 --- a/util/testutil/testutils.go +++ b/util/testutil/testutils.go @@ -11,7 +11,6 @@ import ( "testing" "time" - "github.com/cockroachdb/errors" "github.com/data-preservation-programs/singularity/database" "github.com/data-preservation-programs/singularity/model" "github.com/google/uuid" From 003c7f41ed469bfa407c179a632cf3cb4270fa48 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Sun, 26 Oct 2025 10:45:58 +0000 Subject: [PATCH 12/13] remove unused RandomLetterString --- util/testutil/testutils.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/util/testutil/testutils.go b/util/testutil/testutils.go index 7e544846..55bc58c2 100644 --- a/util/testutil/testutils.go +++ b/util/testutil/testutils.go @@ -4,7 +4,6 @@ import ( "context" "crypto/rand" "io" - rand2 "math/rand" "os" "os/exec" "strings" @@ -38,17 +37,6 @@ func GenerateRandomBytes(n int) []byte { return b } -func RandomLetterString(length int) string { - const charset = "abcdefghijklmnopqrstuvwxyz" - - b := make([]byte, length) - for i := range b { - //nolint:gosec - b[i] = charset[rand2.Intn(len(charset))] - } - return string(b) -} - // GenerateUniqueName creates a unique name for testing by combining a prefix with a UUID suffix func GenerateUniqueName(prefix string) string { return prefix + "-" + strings.ReplaceAll(uuid.New().String(), "-", "") From d6211dd778e460a784e0fff1514a6ac9399a7f5a Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Sun, 26 Oct 2025 11:53:27 +0000 Subject: [PATCH 13/13] init DB with UTF-8 --- util/testutil/testutils.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/testutil/testutils.go b/util/testutil/testutils.go index 55bc58c2..600479aa 100644 --- a/util/testutil/testutils.go +++ b/util/testutil/testutils.go @@ -85,8 +85,8 @@ func getTestDB(t *testing.T, dialect string) (db *gorm.DB, closer io.Closer, con // Create database using shell commands to avoid driver transaction issues switch dialect { case "postgres": - // Use createdb command for PostgreSQL - cmd := exec.Command("createdb", "-h", "localhost", "-p", os.Getenv("PGPORT"), "-U", "singularity", dbName) + // Use createdb command for PostgreSQL with UTF-8 encoding + cmd := exec.Command("createdb", "-h", "localhost", "-p", os.Getenv("PGPORT"), "-U", "singularity", "-E", "UTF8", dbName) output, err := cmd.CombinedOutput() if err != nil { t.Logf("Failed to create PostgreSQL database %s: %v, output: %s", dbName, err, string(output)) @@ -94,9 +94,9 @@ func getTestDB(t *testing.T, dialect string) (db *gorm.DB, closer io.Closer, con } t.Logf("Created PostgreSQL database %s", dbName) case "mysql": - // Use mysql command for MySQL + // Use mysql command for MySQL with UTF-8 character set socket := os.Getenv("MYSQL_SOCKET") - cmd := exec.Command("mariadb", "--socket="+socket, "-usingularity", "-psingularity", "-e", "CREATE DATABASE "+dbName) + cmd := exec.Command("mariadb", "--socket="+socket, "-usingularity", "-psingularity", "-e", "CREATE DATABASE "+dbName+" CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci") output, err := cmd.CombinedOutput() if err != nil { t.Logf("Failed to create MySQL database %s: %v, output: %s", dbName, err, string(output))