From 4059d02952daf688ef402114864ddb0a30d21aea Mon Sep 17 00:00:00 2001 From: Thomas Tacquet Date: Sat, 27 Dec 2025 11:53:15 +0100 Subject: [PATCH 1/4] feat(jobs): jobs-block-snapshot-s3-archiver --- jobs/block-snapshot-s3-archiver/Dockerfile | 27 +++++ jobs/block-snapshot-s3-archiver/README.md | 44 +++++++ jobs/block-snapshot-s3-archiver/config.go | 62 ++++++++++ jobs/block-snapshot-s3-archiver/go.mod | 31 +++++ jobs/block-snapshot-s3-archiver/go.sum | 62 ++++++++++ jobs/block-snapshot-s3-archiver/main.go | 130 +++++++++++++++++++++ 6 files changed, 356 insertions(+) create mode 100644 jobs/block-snapshot-s3-archiver/Dockerfile create mode 100644 jobs/block-snapshot-s3-archiver/README.md create mode 100644 jobs/block-snapshot-s3-archiver/config.go create mode 100644 jobs/block-snapshot-s3-archiver/go.mod create mode 100644 jobs/block-snapshot-s3-archiver/go.sum create mode 100644 jobs/block-snapshot-s3-archiver/main.go diff --git a/jobs/block-snapshot-s3-archiver/Dockerfile b/jobs/block-snapshot-s3-archiver/Dockerfile new file mode 100644 index 0000000..9bd7d51 --- /dev/null +++ b/jobs/block-snapshot-s3-archiver/Dockerfile @@ -0,0 +1,27 @@ +# Build stage +FROM golang:1.25-alpine AS builder + +WORKDIR /app + +# Copy required files +COPY go.mod go.sum ./ +RUN go mod download + +COPY *.go ./ + +# Build the executable +RUN CGO_ENABLED=0 GOOS=linux go build -o /jobs-snapshot-s3 + +# Final stage +FROM alpine:latest + +WORKDIR /app + +# Install CA certificates for HTTPS +RUN apk --no-cache add ca-certificates + +# Copy the binary from the builder stage +COPY --from=builder /jobs-snapshot-s3 /app/jobs-snapshot-s3 + +# Run the executable +ENTRYPOINT [ "/app/jobs-snapshot-s3" ] diff --git a/jobs/block-snapshot-s3-archiver/README.md b/jobs/block-snapshot-s3-archiver/README.md new file mode 100644 index 0000000..7d261a8 --- /dev/null +++ b/jobs/block-snapshot-s3-archiver/README.md @@ -0,0 +1,44 @@ +# Scaleway Instance Snapshot Archiver + +Automated serverless job to archive Scaleway Instance snapshots to Object Storage S3. + +## Features + +- **Automated Export**: Finds available snapshots and exports them to an S3 bucket in `.qcow2` format. +- **Cost Optimization**: Deletes the source snapshot after successful export to reduce storage costs. +- **Idempotent**: Skips snapshots that are already archived in the bucket. +- **Serverless Ready**: Designed for [Scaleway Serverless Jobs](https://www.scaleway.com/en/serverless-jobs/). + +## Configuration + +Configure the job using environment variables: + +| Variable | Description | +|---|---| +| `SCW_DEFAULT_ORGANIZATION_ID` | Organization ID (Legacy). | +| `SCW_DEFAULT_PROJECT_ID` | Project ID (Recommended resource grouping). | +| `SCW_ACCESS_KEY` | IAM Access Key. | +| `SCW_SECRET_KEY` | IAM Secret Key. | +| `SCW_ZONE` | Zone of the snapshots (e.g., `fr-par-1`). | +| `SCW_BUCKET_NAME` | S3 Bucket name for archives. | +| `SCW_BUCKET_ENDPOINT` | S3 Endpoint (e.g., `s3.fr-par.scw.cloud`). | + +## Usage + +### 1. Build + +```bash +docker build -t snapshot-archiver . +``` + +### 2. Run Locally + +Ensure all environment variables are set, then run: + +```bash +go run . +``` + +### 3. Deploy + +Push the image to your container registry and create a Serverless Job definition pointing to it with the required environment variables. \ No newline at end of file diff --git a/jobs/block-snapshot-s3-archiver/config.go b/jobs/block-snapshot-s3-archiver/config.go new file mode 100644 index 0000000..960347a --- /dev/null +++ b/jobs/block-snapshot-s3-archiver/config.go @@ -0,0 +1,62 @@ +package main + +import ( + "fmt" + "os" + + "github.com/scaleway/scaleway-sdk-go/scw" +) + +// Environment variable constants +const ( + envOrgID = "SCW_DEFAULT_ORGANIZATION_ID" + envAccessKey = "SCW_ACCESS_KEY" + envSecretKey = "SCW_SECRET_KEY" + envProjectID = "SCW_DEFAULT_PROJECT_ID" + envZone = "SCW_ZONE" + envBucket = "SCW_BUCKET_NAME" + envBucketEndpoint = "SCW_BUCKET_ENDPOINT" +) + +type Config struct { + OrgID string + AccessKey string + SecretKey string + ProjectID string + Zone scw.Zone + BucketName string + BucketEndpoint string +} + +func LoadConfig() (*Config, error) { + // Mandatory variables + vars := map[string]*string{ + envAccessKey: new(string), + envSecretKey: new(string), + envProjectID: new(string), + envZone: new(string), + envBucket: new(string), + envBucketEndpoint: new(string), + } + + // Optional variables + orgID := os.Getenv(envOrgID) + + for envKey, valPtr := range vars { + val := os.Getenv(envKey) + if val == "" { + return nil, fmt.Errorf("missing environment variable %s", envKey) + } + *valPtr = val + } + + return &Config{ + OrgID: orgID, + AccessKey: *vars[envAccessKey], + SecretKey: *vars[envSecretKey], + ProjectID: *vars[envProjectID], + Zone: scw.Zone(*vars[envZone]), + BucketName: *vars[envBucket], + BucketEndpoint: *vars[envBucketEndpoint], + }, nil +} diff --git a/jobs/block-snapshot-s3-archiver/go.mod b/jobs/block-snapshot-s3-archiver/go.mod new file mode 100644 index 0000000..c420369 --- /dev/null +++ b/jobs/block-snapshot-s3-archiver/go.mod @@ -0,0 +1,31 @@ +module github.com/scaleway/serverless-examples/jobs/snapshot-s3-archiver + +go 1.25 + +require ( + github.com/minio/minio-go/v7 v7.0.95 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tinylib/msgp v1.4.0 // indirect + golang.org/x/crypto v0.42.0 // indirect + golang.org/x/net v0.44.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/jobs/block-snapshot-s3-archiver/go.sum b/jobs/block-snapshot-s3-archiver/go.sum new file mode 100644 index 0000000..79cac08 --- /dev/null +++ b/jobs/block-snapshot-s3-archiver/go.sum @@ -0,0 +1,62 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35 h1:8xfn1RzeI9yoCUuEwDy08F+No6PcKZGEDOQ6hrRyLts= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35/go.mod h1:47B1d/YXmSAxlJxUJxClzHR6b3T4M1WyCvwENPQNBWc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.4.0 h1:SYOeDRiydzOw9kSiwdYp9UcBgPFtLU2WDHaJXyHruf8= +github.com/tinylib/msgp v1.4.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/jobs/block-snapshot-s3-archiver/main.go b/jobs/block-snapshot-s3-archiver/main.go new file mode 100644 index 0000000..d76262d --- /dev/null +++ b/jobs/block-snapshot-s3-archiver/main.go @@ -0,0 +1,130 @@ +package main + +import ( + "context" + "log/slog" + "os" + "slices" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/scaleway/scaleway-sdk-go/api/block/v1" + "github.com/scaleway/scaleway-sdk-go/scw" +) + +const snapshotExtension = ".qcow2" + +func main() { + // Configure valid JSON logger + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + slog.SetDefault(logger) + + // Load configuration + cfg, err := LoadConfig() + if err != nil { + slog.Error("Failed to load configuration", "error", err) + os.Exit(1) + } + + // Create Scaleway client using the implementation in config + client, err := scw.NewClient( + scw.WithDefaultOrganizationID(cfg.OrgID), + scw.WithAuth(cfg.AccessKey, cfg.SecretKey), + scw.WithDefaultProjectID(cfg.ProjectID), + scw.WithDefaultZone(cfg.Zone), + ) + if err != nil { + slog.Error("Failed to create Scaleway client", "error", err) + os.Exit(1) + } + + slog.Info("Initializing block API...") + blockAPI := block.NewAPI(client) + + slog.Info("Reading all snapshots for the project...") + snapList, err := blockAPI.ListSnapshots(&block.ListSnapshotsRequest{ + Zone: cfg.Zone, + ProjectID: &cfg.ProjectID, + }, scw.WithAllPages()) + if err != nil { + slog.Error("Failed to list snapshots", "error", err) + os.Exit(1) + } + + slog.Info("Reading all snapshots already in the bucket...") + filesInBucket, err := listBucketFiles(cfg) + if err != nil { + slog.Error("Failed to list bucket files", "error", err) + os.Exit(1) + } + + processSnapshots(blockAPI, cfg, snapList.Snapshots, filesInBucket) +} + +func listBucketFiles(cfg *Config) ([]string, error) { + minioClient, err := minio.New(cfg.BucketEndpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: true, + }) + if err != nil { + return nil, err + } + + ctx := context.Background() + var files []string + + opts := minio.ListObjectsOptions{ + Recursive: false, + WithMetadata: true, + } + + for object := range minioClient.ListObjects(ctx, cfg.BucketName, opts) { + if object.Err != nil { + return nil, object.Err + } + files = append(files, object.Key) + } + + return files, nil +} + +func processSnapshots(api *block.API, cfg *Config, snapshots []*block.Snapshot, filesInBucket []string) { + for _, snapshot := range snapshots { + logger := slog.With("snapshot_id", snapshot.ID, "snapshot_name", snapshot.Name) + + logger.Info("Checking snapshot") + + if snapshot.Status != block.SnapshotStatusAvailable { + logger.Info("Skipping snapshot (not available)", "status", snapshot.Status.String()) + continue + } + + filename := snapshot.Name + snapshotExtension + + if slices.Contains(filesInBucket, filename) { + logger.Info("File already exists in bucket, deleting local snapshot") + err := api.DeleteSnapshot(&block.DeleteSnapshotRequest{ + SnapshotID: snapshot.ID, + Zone: snapshot.Zone, + }) + if err != nil { + logger.Error("Failed to delete snapshot", "error", err) + } + continue + } + + logger.Info("Exporting snapshot to bucket") + snap, err := api.ExportSnapshotToObjectStorage(&block.ExportSnapshotToObjectStorageRequest{ + SnapshotID: snapshot.ID, + Bucket: cfg.BucketName, + Key: filename, + Zone: snapshot.Zone, + }) + if err != nil { + logger.Error("Failed to export snapshot", "error", err) + continue + } + + logger.Info("Successfully started export", "task_id", snap.ID, "bucket", cfg.BucketName) + } +} From 491bae155168a7924d2c3840f40293e30f57f9d3 Mon Sep 17 00:00:00 2001 From: Thomas Tacquet Date: Mon, 29 Dec 2025 17:18:09 +0100 Subject: [PATCH 2/4] improve documentation --- jobs/block-snapshot-s3-archiver/README.md | 72 +++++++++++++++++------ 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/jobs/block-snapshot-s3-archiver/README.md b/jobs/block-snapshot-s3-archiver/README.md index 7d261a8..64365a3 100644 --- a/jobs/block-snapshot-s3-archiver/README.md +++ b/jobs/block-snapshot-s3-archiver/README.md @@ -1,6 +1,18 @@ # Scaleway Instance Snapshot Archiver -Automated serverless job to archive Scaleway Instance snapshots to Object Storage S3. +Automated Serverless Job to archive Scaleway Block Storage snapshots to Object Storage S3. + +## Overview + +This tool automatically finds available snapshots of Scaleway Block Storage volumes, exports them to a specified S3 bucket in `.qcow2` format, and deletes the original snapshot to optimize storage costs. It's designed to run as a Serverless Job on Scaleway and skips snapshots that have already been archived. + +The main logic is implemented in `main.go`, which: +1. Loads configuration from environment variables. +2. Connects to Scaleway APIs using the Scaleway SDK. +3. Lists all available snapshots in the project. +4. Checks the target S3 bucket for already-archived snapshots. +5. Exports new snapshots to the bucket. +6. Deletes successfully exported snapshots to reduce storage costs. ## Features @@ -9,13 +21,42 @@ Automated serverless job to archive Scaleway Instance snapshots to Object Storag - **Idempotent**: Skips snapshots that are already archived in the bucket. - **Serverless Ready**: Designed for [Scaleway Serverless Jobs](https://www.scaleway.com/en/serverless-jobs/). -## Configuration +## Step 1 : Build and push to Container registry + +Serverless Jobs, like Serverless Containers (which are suited for HTTP applications), works +with containers. So first, use your terminal reach this folder and run the following commands: + +```shell +# First command is to login to container registry, you can find it in Scaleway console +docker login rg.fr-par.scw.cloud/block-snapshot-s3-archiver -u nologin --password-stdin <<< "$SCW_SECRET_KEY" + +# Here we build the image to push +docker buildx build --platform linux/amd64 -t rg.fr-par.scw.cloud/block-snapshot-s3-archiver/block-snapshot-s3-archiver:v1 . + +# Push the image online to be used on Serverless Jobs +docker push rg.fr-par.scw.cloud/block-snapshot-s3-archiver/block-snapshot-s3-archiver:v1 +``` +> [!TIP] +> As we do not expose a web server and we do not require features such as auto-scaling, Serverless Jobs are perfect for this use case. -Configure the job using environment variables: +To check if everyting is ok, on the Scaleway Console you can verify if your tag is present in Container Registry. + +## Step 2: Creating the Job Definition + +On Scaleway Console on the following link you can create a new Job Definition: https://console.scaleway.com/serverless-jobs/jobs/create?region=fr-par + +1. On Container image, select the image you created in the step before. +2. You can set the job definition name name to something clear. +3. Regarding the resources you can keep the default values, this job is fast and do not require specific compute power or memory. +4. To schedule your job for example every night at 2am, you can set the cron to `0 2 * * *`. +5. Important: advanced option, you need to set the following environment variables: + +> [!TIP] +> For sensitive data like `SCW_ACCESS_KEY` and `SCW_SECRET_KEY` we recommend to inject them via Secret Manager, [more info here](https://www.scaleway.com/en/docs/serverless/jobs/how-to/reference-secret-in-job/). | Variable | Description | |---|---| -| `SCW_DEFAULT_ORGANIZATION_ID` | Organization ID (Legacy). | +| `SCW_DEFAULT_ORGANIZATION_ID` | Organization ID . | | `SCW_DEFAULT_PROJECT_ID` | Project ID (Recommended resource grouping). | | `SCW_ACCESS_KEY` | IAM Access Key. | | `SCW_SECRET_KEY` | IAM Secret Key. | @@ -23,22 +64,19 @@ Configure the job using environment variables: | `SCW_BUCKET_NAME` | S3 Bucket name for archives. | | `SCW_BUCKET_ENDPOINT` | S3 Endpoint (e.g., `s3.fr-par.scw.cloud`). | -## Usage +* Then click "create job" -### 1. Build +## Step 3: Run the job -```bash -docker build -t snapshot-archiver . -``` - -### 2. Run Locally +On your created Job Definition, just click the button "Run Job" and within seconds it should be successful. -Ensure all environment variables are set, then run: +## Troubleshooting -```bash -go run . -``` +If your Job Run state goes in error, you can use the "Logs" tab in Scaleway Console to get more informations about the error. -### 3. Deploy +# Additional content -Push the image to your container registry and create a Serverless Job definition pointing to it with the required environment variables. \ No newline at end of file +- [Jobs Documentation](https://www.scaleway.com/en/docs/serverless/jobs/how-to/create-job-from-scaleway-registry/) +- [Other methods to deploy Jobs](https://www.scaleway.com/en/docs/serverless/jobs/reference-content/deploy-job/) +- [Secret key / access key doc](https://www.scaleway.com/en/docs/identity-and-access-management/iam/how-to/create-api-keys/) +- [CRON schedule help](https://www.scaleway.com/en/docs/serverless/jobs/reference-content/cron-schedules/) From d78218a385c306a6d8d98dda0a10c50179fc08f3 Mon Sep 17 00:00:00 2001 From: Thomas Tacquet Date: Mon, 29 Dec 2025 17:20:15 +0100 Subject: [PATCH 3/4] update base readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7ead77c..218f521 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ Table of Contents: | **[Instance Snapshot Cleaner](jobs/instances-snapshot-cleaner/README.md)**
Use Serverless Jobs to clean old instances snapshots | Go | [Console] | | **[Registry Tag Cleaner](jobs/registry-version-based-retention/README.md)**
Use Serverless Jobs to keep a desired amount of tags for each image | Go | [Console] | | **[Registry Empty Image Cleaner](jobs/registry-empty-ressource-cleaner/README.md)**
Use Serverless Jobs to clean container registry empty namespaces and images | Go | [Console] | +| **[Block snapshots to S3](jobs/block-snapshot-s3-archiver/README.md)**
Use Serverless Jobs to move Block Storage snapshots to S3 Object Storage | Go | [Console] | ### 💬 Messaging and Queueing From 68e831e1d68f22e16de46422e90f6451a544c7b5 Mon Sep 17 00:00:00 2001 From: Thomas Tacquet Date: Mon, 29 Dec 2025 17:30:00 +0100 Subject: [PATCH 4/4] readme update --- jobs/block-snapshot-s3-archiver/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/block-snapshot-s3-archiver/README.md b/jobs/block-snapshot-s3-archiver/README.md index 64365a3..88b3633 100644 --- a/jobs/block-snapshot-s3-archiver/README.md +++ b/jobs/block-snapshot-s3-archiver/README.md @@ -1,4 +1,4 @@ -# Scaleway Instance Snapshot Archiver +# Scaleway Block Storage Snapshot Archiver Automated Serverless Job to archive Scaleway Block Storage snapshots to Object Storage S3.