From defc305abfc3390f804499960995e2a570fac71d Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Fri, 23 Jan 2026 15:08:09 -0600 Subject: [PATCH 1/8] feat: add user metadata support for uploads and RLS policies --- src/http/routes/object/getSignedUploadURL.ts | 10 ++++++ src/http/routes/tus/lifecycle.ts | 17 ++++++++- src/storage/object.ts | 5 ++- src/storage/protocols/s3/s3-handler.ts | 38 +++++++++++--------- src/storage/uploader.ts | 14 +++++--- src/test/cdn.test.ts | 2 +- src/test/rls.test.ts | 28 ++++++++++++--- src/test/rls_tests.yaml | 20 +++++++++++ src/test/scanner.test.ts | 4 +-- 9 files changed, 108 insertions(+), 30 deletions(-) diff --git a/src/http/routes/object/getSignedUploadURL.ts b/src/http/routes/object/getSignedUploadURL.ts index 5a68afa30..b5248237a 100644 --- a/src/http/routes/object/getSignedUploadURL.ts +++ b/src/http/routes/object/getSignedUploadURL.ts @@ -4,6 +4,7 @@ import { createDefaultSchema } from '../../routes-helper' import { AuthenticatedRequest } from '../../types' import { getConfig } from '../../../config' import { ROUTE_OPERATIONS } from '../operations' +import { parseUserMetadata } from '../../../storage/uploader' const { uploadSignedUrlExpirationTime } = getConfig() @@ -69,10 +70,19 @@ export default async function routes(fastify: FastifyInstance) { const urlPath = `${bucketName}/${objectName}` + let userMetadata: Record | undefined + + const customMd = request.headers['x-metadata'] + + if (typeof customMd === 'string') { + userMetadata = parseUserMetadata(customMd) + } + const signedUpload = await request.storage .from(bucketName) .signUploadObjectUrl(objectName, urlPath as string, uploadSignedUrlExpirationTime, owner, { upsert: request.headers['x-upsert'] === 'true', + userMetadata: userMetadata, }) return response.status(200).send({ url: signedUpload.url, token: signedUpload.token }) diff --git a/src/http/routes/tus/lifecycle.ts b/src/http/routes/tus/lifecycle.ts index a34716485..02ea4f5c6 100644 --- a/src/http/routes/tus/lifecycle.ts +++ b/src/http/routes/tus/lifecycle.ts @@ -1,6 +1,6 @@ import http from 'http' import { BaseLogger } from 'pino' -import { Upload } from '@tus/server' +import { Metadata, Upload } from '@tus/server' import { randomUUID } from 'crypto' import { TenantConnection } from '@internal/database' import { ERRORS, isRenderableError } from '@internal/errors' @@ -63,6 +63,20 @@ export async function onIncomingRequest(rawReq: Request, id: string) { req.upload.resources = [`${uploadID.bucket}/${uploadID.objectName}`] + // following same metadata parsing logic as @tus/server PostHandler so it + // it matches the value on Upload.metadata when inserted in storage.objects + // link: https://github.com/tus/tus-node-server/blob/1a6482a7a55e1587bda8c6887250f36cf9d606bd/packages/server/src/handlers/PostHandler.ts#L46 + let customMd: Record | undefined = undefined + const uploadMetadataHeader = req.headers['upload-metadata'] + + if (uploadMetadataHeader && typeof uploadMetadataHeader === 'string') { + try { + customMd = Metadata.parse(uploadMetadataHeader) + } catch (e) { + req.log.warn({ error: e }, 'Failed to parse user metadata') + } + } + // Handle signed url requests if (req.url?.startsWith(`/upload/resumable/sign`)) { const signature = req.headers['x-signature'] @@ -97,6 +111,7 @@ export async function onIncomingRequest(rawReq: Request, id: string) { bucketId: uploadID.bucket, objectName: uploadID.objectName, isUpsert: isUpsert, + userMetadata: customMd, }) } diff --git a/src/storage/object.ts b/src/storage/object.ts index 4818ad131..5ec845728 100644 --- a/src/storage/object.ts +++ b/src/storage/object.ts @@ -98,6 +98,7 @@ export class ObjectStorage { owner: file.owner, isUpsert: Boolean(file.isUpsert), signal: file.signal, + userMetadata: uploadRequest.userMetadata, }) } @@ -339,6 +340,7 @@ export class ObjectStorage { objectName: destinationKey, owner, isUpsert: upsert, + userMetadata: userMetadata, }) try { @@ -792,7 +794,7 @@ export class ObjectStorage { url: string, expiresIn: number, owner?: string, - options?: { upsert?: boolean } + options?: { upsert?: boolean; userMetadata?: Record } ) { // check if user has INSERT permissions await this.uploader.canUpload({ @@ -800,6 +802,7 @@ export class ObjectStorage { objectName, owner, isUpsert: options?.upsert ?? false, + userMetadata: options?.userMetadata, }) const { urlSigningKey } = await getJwtSecret(this.db.tenantId) diff --git a/src/storage/protocols/s3/s3-handler.ts b/src/storage/protocols/s3/s3-handler.ts index a6883a577..9ae4dba53 100644 --- a/src/storage/protocols/s3/s3-handler.ts +++ b/src/storage/protocols/s3/s3-handler.ts @@ -413,6 +413,7 @@ export class S3ProtocolHandler { objectName: command.Key as string, isUpsert: true, owner: this.owner, + userMetadata: command.Metadata, }) const uploadId = await this.storage.backend.createMultiPartUpload( @@ -470,17 +471,18 @@ export class S3ProtocolHandler { throw ERRORS.InvalidUploadId() } + const multiPartUpload = await this.storage.db + .asSuperUser() + .findMultipartUpload(UploadId, 'id,version,user_metadata') + await uploader.canUpload({ bucketId: Bucket as string, objectName: Key as string, isUpsert: true, owner: this.owner, + userMetadata: multiPartUpload.user_metadata || undefined, }) - const multiPartUpload = await this.storage.db - .asSuperUser() - .findMultipartUpload(UploadId, 'id,version,user_metadata') - const parts = command.MultipartUpload?.Parts || [] if (parts.length === 0) { @@ -578,15 +580,17 @@ export class S3ProtocolHandler { const maxFileSize = await getFileSizeLimit(this.storage.db.tenantId, bucket?.file_size_limit) const uploader = new Uploader(this.storage.backend, this.storage.db, this.storage.location) + + const multipart = await this.shouldAllowPartUpload(UploadId, ContentLength, maxFileSize) + await uploader.canUpload({ bucketId: Bucket as string, objectName: Key as string, owner: this.owner, isUpsert: true, + userMetadata: multipart.user_metadata || undefined, }) - const multipart = await this.shouldAllowPartUpload(UploadId, ContentLength, maxFileSize) - if (signal?.aborted) { throw ERRORS.AbortedTerminate('UploadPart aborted') } @@ -695,9 +699,9 @@ export class S3ProtocolHandler { cacheControl: command.CacheControl!, mimeType: command.ContentType!, isTruncated: options.isTruncated, - userMetadata: command.Metadata, }, objectName: command.Key as string, + userMetadata: command.Metadata, owner: this.owner, isUpsert: true, uploadType: 's3', @@ -735,7 +739,7 @@ export class S3ProtocolHandler { const multipart = await this.storage.db .asSuperUser() - .findMultipartUpload(UploadId, 'id,version') + .findMultipartUpload(UploadId, 'id,version,user_metadata') const uploader = new Uploader(this.storage.backend, this.storage.db, this.storage.location) await uploader.canUpload({ @@ -743,6 +747,7 @@ export class S3ProtocolHandler { objectName: Key, owner: this.owner, isUpsert: true, + userMetadata: multipart.user_metadata || undefined, }) await this.storage.backend.abortMultipartUpload( @@ -1233,13 +1238,6 @@ export class S3ProtocolHandler { const uploader = new Uploader(this.storage.backend, this.storage.db, this.storage.location) - await uploader.canUpload({ - bucketId: Bucket, - objectName: Key, - owner: this.owner, - isUpsert: true, - }) - const [destinationBucket] = await this.storage.db.asSuperUser().withTransaction(async (db) => { return Promise.all([ db.findBucketById(Bucket, 'file_size_limit'), @@ -1253,6 +1251,14 @@ export class S3ProtocolHandler { const multipart = await this.shouldAllowPartUpload(UploadId, Number(copySize), maxFileSize) + await uploader.canUpload({ + bucketId: Bucket, + objectName: Key, + owner: this.owner, + isUpsert: true, + userMetadata: multipart.user_metadata || undefined, + }) + const uploadPart = await this.storage.backend.uploadPartCopy( storageS3Bucket, this.storage.location.getKeyLocation({ @@ -1324,7 +1330,7 @@ export class S3ProtocolHandler { return this.storage.db.asSuperUser().withTransaction(async (db) => { const multipart = await db.findMultipartUpload( uploadId, - 'in_progress_size,version,upload_signature', + 'in_progress_size,version,upload_signature,user_metadata', { forUpdate: true, } diff --git a/src/storage/uploader.ts b/src/storage/uploader.ts index c30ff4ace..7a862e290 100644 --- a/src/storage/uploader.ts +++ b/src/storage/uploader.ts @@ -22,13 +22,13 @@ interface FileUpload { cacheControl: string isTruncated: () => boolean xRobotsTag?: string - userMetadata?: Record } export interface UploadRequest { bucketId: string objectName: string file: FileUpload + userMetadata: Record | undefined owner?: string isUpsert?: boolean uploadType?: 'standard' | 's3' | 'resumable' @@ -48,7 +48,9 @@ export class Uploader { private readonly location: StorageObjectLocator ) {} - async canUpload(options: Pick) { + async canUpload( + options: Pick + ) { const shouldCreateObject = !options.isUpsert if (shouldCreateObject) { @@ -58,6 +60,7 @@ export class Uploader { name: options.objectName, version: '1', owner: options.owner, + user_metadata: options.userMetadata, }) }) } else { @@ -67,6 +70,7 @@ export class Uploader { name: options.objectName, version: '1', owner: options.owner, + user_metadata: options.userMetadata, }) }) } @@ -127,7 +131,7 @@ export class Uploader { ...request, version, objectMetadata: objectMetadata, - userMetadata: { ...file.userMetadata }, + userMetadata: { ...request.userMetadata }, }) } catch (e) { await ObjectAdminDelete.send({ @@ -310,7 +314,9 @@ export async function fileUploadFromRequest( allowedMimeTypes?: string[] objectName: string } -): Promise { +): Promise< + FileUpload & { maxFileSize: number; userMetadata: Record | undefined } +> { const contentType = request.headers['content-type'] const xRobotsTag = request.headers['x-robots-tag'] as string | undefined diff --git a/src/test/cdn.test.ts b/src/test/cdn.test.ts index 0bd260d17..cef80ebee 100644 --- a/src/test/cdn.test.ts +++ b/src/test/cdn.test.ts @@ -94,12 +94,12 @@ describe('CDN Cache Manager', () => { await storageHook.storage.from(bucketName).uploadNewObject({ isUpsert: true, objectName, + userMetadata: {}, file: { body: Readable.from(Buffer.from('test')), cacheControl: 'public, max-age=31536000', mimeType: 'text/plain', isTruncated: () => false, - userMetadata: {}, }, }) diff --git a/src/test/rls.test.ts b/src/test/rls.test.ts index f6084ee46..f2bd5a233 100644 --- a/src/test/rls.test.ts +++ b/src/test/rls.test.ts @@ -58,6 +58,7 @@ interface TestCaseAssert { useExistingBucketName?: string role?: string policies?: string[] + userMetadata?: Record status: number error?: string } @@ -236,6 +237,7 @@ describe('RLS policies', () => { bucket: bucketName, objectName: objectName, jwt: assert.role === 'service' ? await serviceKeyAsync : jwt, + userMetadata: assert.userMetadata, }) console.log( @@ -294,15 +296,20 @@ describe('RLS policies', () => { async function runOperation( operation: TestCaseAssert['operation'], - options: { bucket: string; jwt: string; objectName: string } + options: { + bucket: string + jwt: string + objectName: string + userMetadata?: Record + } ) { - const { jwt, bucket, objectName } = options + const { jwt, bucket, objectName, userMetadata } = options switch (operation) { case 'upload': - return uploadFile(bucket, objectName, jwt) + return uploadFile(bucket, objectName, jwt, false, userMetadata) case 'upload.upsert': - return uploadFile(bucket, objectName, jwt, true) + return uploadFile(bucket, objectName, jwt, true, userMetadata) case 'bucket.list': return appInstance.inject({ method: 'GET', @@ -454,10 +461,21 @@ async function createPolicy(db: Knex, policy: Policy) { return Promise.all(created) } -async function uploadFile(bucket: string, fileName: string, jwt: string, upsert?: boolean) { +async function uploadFile( + bucket: string, + fileName: string, + jwt: string, + upsert?: boolean, + userMetadata?: Record +) { const testFile = fs.createReadStream(path.resolve(__dirname, 'assets', 'sadcat.jpg')) const form = new FormData() form.append('file', testFile) + + if (userMetadata) { + form.append('metadata', JSON.stringify(userMetadata)) + } + const headers = Object.assign({}, form.getHeaders(), { authorization: `Bearer ${jwt}`, ...(upsert ? { 'x-upsert': 'true' } : {}), diff --git a/src/test/rls_tests.yaml b/src/test/rls_tests.yaml index 150c43a70..d4952e63d 100644 --- a/src/test/rls_tests.yaml +++ b/src/test/rls_tests.yaml @@ -47,6 +47,12 @@ policies: permissions: ['delete'] content: "USING(owner = '{{uid}}')" + - name: insert_with_metadata_check + tables: ['storage.objects'] + roles: ['authenticated'] + permissions: ['insert'] + content: "WITH CHECK(user_metadata->>'department' = 'engineering')" + tests: - description: 'Will only able to read objects' policies: @@ -475,3 +481,17 @@ tests: - operation: bucket.delete status: 400 error: 'Bucket not found' + + - description: 'Will only upload files with correct user metadata' + policies: + - insert_with_metadata_check + asserts: + - operation: upload + objectName: 'test_file.jpg' + userMetadata: + department: 'engineering' + status: 200 + + - operation: upload + status: 400 + error: 'new row violates row-level security policy' diff --git a/src/test/scanner.test.ts b/src/test/scanner.test.ts index ab4cbbe3c..5c0cd5b1c 100644 --- a/src/test/scanner.test.ts +++ b/src/test/scanner.test.ts @@ -24,11 +24,11 @@ describe('ObjectScanner', () => { bucketId: bucket.id, objectName: randomUUID() + `-test-${i}.text`, uploadType: 'standard', + userMetadata: {}, file: { body: Readable.from(Buffer.from('test')), mimeType: 'text/plain', cacheControl: 'no-cache', - userMetadata: {}, isTruncated: () => false, }, }) @@ -91,11 +91,11 @@ describe('ObjectScanner', () => { bucketId: bucket.id, objectName: randomUUID() + `-test-${i}.text`, uploadType: 'standard', + userMetadata: {}, file: { body: Readable.from(Buffer.from('test')), mimeType: 'text/plain', cacheControl: 'no-cache', - userMetadata: {}, isTruncated: () => false, }, }) From 1c4d6ce285252dc848bf46b89162aff0dc7641bb Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Mon, 26 Jan 2026 13:13:28 -0600 Subject: [PATCH 2/8] fix: update metadata parsing logic to handle user metadata correctly --- src/http/routes/tus/lifecycle.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/http/routes/tus/lifecycle.ts b/src/http/routes/tus/lifecycle.ts index 02ea4f5c6..aa2806b9d 100644 --- a/src/http/routes/tus/lifecycle.ts +++ b/src/http/routes/tus/lifecycle.ts @@ -63,15 +63,15 @@ export async function onIncomingRequest(rawReq: Request, id: string) { req.upload.resources = [`${uploadID.bucket}/${uploadID.objectName}`] - // following same metadata parsing logic as @tus/server PostHandler so it - // it matches the value on Upload.metadata when inserted in storage.objects - // link: https://github.com/tus/tus-node-server/blob/1a6482a7a55e1587bda8c6887250f36cf9d606bd/packages/server/src/handlers/PostHandler.ts#L46 - let customMd: Record | undefined = undefined + let customMd: Record | undefined = undefined const uploadMetadataHeader = req.headers['upload-metadata'] if (uploadMetadataHeader && typeof uploadMetadataHeader === 'string') { try { - customMd = Metadata.parse(uploadMetadataHeader) + const parsedMetadata = Metadata.parse(uploadMetadataHeader) + if (parsedMetadata?.metadata) { + customMd = JSON.parse(parsedMetadata.metadata) + } } catch (e) { req.log.warn({ error: e }, 'Failed to parse user metadata') } From 7303d1c5027d3cb53d504b2078e0614ee55872e2 Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Mon, 2 Feb 2026 19:22:51 -0600 Subject: [PATCH 3/8] feat: add file metadata support for uploads and RLS policies --- .../0056-s3-multipart-uploads-metadata.sql | 1 + src/http/routes/object/getSignedUploadURL.ts | 10 ++ src/http/routes/tus/lifecycle.ts | 8 ++ src/internal/database/migrations/types.ts | 121 +++++++++--------- src/storage/database/adapter.ts | 3 +- src/storage/database/knex.ts | 6 +- src/storage/object.ts | 10 +- src/storage/protocols/s3/s3-handler.ts | 16 ++- src/storage/schemas/multipart.ts | 3 + src/storage/uploader.ts | 43 ++++++- src/test/rls.test.ts | 21 ++- src/test/rls_tests.yaml | 41 ++++++ 12 files changed, 205 insertions(+), 78 deletions(-) create mode 100644 migrations/tenant/0056-s3-multipart-uploads-metadata.sql diff --git a/migrations/tenant/0056-s3-multipart-uploads-metadata.sql b/migrations/tenant/0056-s3-multipart-uploads-metadata.sql new file mode 100644 index 000000000..ef9496bb8 --- /dev/null +++ b/migrations/tenant/0056-s3-multipart-uploads-metadata.sql @@ -0,0 +1 @@ +ALTER TABLE storage.s3_multipart_uploads ADD COLUMN IF NOT EXISTS metadata jsonb NULL; \ No newline at end of file diff --git a/src/http/routes/object/getSignedUploadURL.ts b/src/http/routes/object/getSignedUploadURL.ts index b5248237a..02159ddab 100644 --- a/src/http/routes/object/getSignedUploadURL.ts +++ b/src/http/routes/object/getSignedUploadURL.ts @@ -21,6 +21,8 @@ const getSignedUploadURLHeadersSchema = { type: 'object', properties: { 'x-upsert': { type: 'string' }, + 'content-type': { type: 'string' }, + 'content-length': { type: 'string' }, authorization: { type: 'string' }, }, required: ['authorization'], @@ -78,11 +80,19 @@ export default async function routes(fastify: FastifyInstance) { userMetadata = parseUserMetadata(customMd) } + const contentType = request.headers['content-type'] + const contentLengthHeader = request.headers['content-length'] + const contentLength = contentLengthHeader ? Number(contentLengthHeader) : undefined + const signedUpload = await request.storage .from(bucketName) .signUploadObjectUrl(objectName, urlPath as string, uploadSignedUrlExpirationTime, owner, { upsert: request.headers['x-upsert'] === 'true', userMetadata: userMetadata, + metadata: { + mimetype: contentType, + contentLength: contentLength, + }, }) return response.status(200).send({ url: signedUpload.url, token: signedUpload.token }) diff --git a/src/http/routes/tus/lifecycle.ts b/src/http/routes/tus/lifecycle.ts index aa2806b9d..2c57a51d7 100644 --- a/src/http/routes/tus/lifecycle.ts +++ b/src/http/routes/tus/lifecycle.ts @@ -106,12 +106,20 @@ export async function onIncomingRequest(rawReq: Request, id: string) { req.upload.storage.location ) + const uploadLength = req.headers['upload-length'] + const contentLength = uploadLength ? Number(uploadLength) : undefined + const contentType = req.headers['content-type'] + await uploader.canUpload({ owner: req.upload.owner, bucketId: uploadID.bucket, objectName: uploadID.objectName, isUpsert: isUpsert, userMetadata: customMd, + metadata: { + mimetype: contentType, + contentLength: contentLength, + }, }) } diff --git a/src/internal/database/migrations/types.ts b/src/internal/database/migrations/types.ts index 35bd839c0..bebd77633 100644 --- a/src/internal/database/migrations/types.ts +++ b/src/internal/database/migrations/types.ts @@ -1,59 +1,62 @@ -export const DBMigration = { - 'create-migrations-table': 0, - initialmigration: 1, - 'storage-schema': 2, - 'pathtoken-column': 3, - 'add-migrations-rls': 4, - 'add-size-functions': 5, - 'change-column-name-in-get-size': 6, - 'add-rls-to-buckets': 7, - 'add-public-to-buckets': 8, - 'fix-search-function': 9, - 'search-files-search-function': 10, - 'add-trigger-to-auto-update-updated_at-column': 11, - 'add-automatic-avif-detection-flag': 12, - 'add-bucket-custom-limits': 13, - 'use-bytes-for-max-size': 14, - 'add-can-insert-object-function': 15, - 'add-version': 16, - 'drop-owner-foreign-key': 17, - add_owner_id_column_deprecate_owner: 18, - 'alter-default-value-objects-id': 19, - 'list-objects-with-delimiter': 20, - 's3-multipart-uploads': 21, - 's3-multipart-uploads-big-ints': 22, - 'optimize-search-function': 23, - 'operation-function': 24, - 'custom-metadata': 25, - 'objects-prefixes': 26, - 'search-v2': 27, - 'object-bucket-name-sorting': 28, - 'create-prefixes': 29, - 'update-object-levels': 30, - 'objects-level-index': 31, - 'backward-compatible-index-on-objects': 32, - 'backward-compatible-index-on-prefixes': 33, - 'optimize-search-function-v1': 34, - 'add-insert-trigger-prefixes': 35, - 'optimise-existing-functions': 36, - 'add-bucket-name-length-trigger': 37, - 'iceberg-catalog-flag-on-buckets': 38, - 'add-search-v2-sort-support': 39, - 'fix-prefix-race-conditions-optimized': 40, - 'add-object-level-update-trigger': 41, - 'rollback-prefix-triggers': 42, - 'fix-object-level': 43, - 'vector-bucket-type': 44, - 'vector-buckets': 45, - 'buckets-objects-grants': 46, - 'iceberg-table-metadata': 47, - 'iceberg-catalog-ids': 48, - 'buckets-objects-grants-postgres': 49, - 'search-v2-optimised': 50, - 'index-backward-compatible-search': 51, - 'drop-not-used-indexes-and-functions': 52, - 'drop-index-lower-name': 53, - 'drop-index-object-level': 54, - 'prevent-direct-deletes': 55, - 'fix-optimized-search-function': 56, -} + + export const DBMigration = { + 'create-migrations-table': 0, + 'initialmigration': 1, + 'storage-schema': 2, + 'pathtoken-column': 3, + 'add-migrations-rls': 4, + 'add-size-functions': 5, + 'change-column-name-in-get-size': 6, + 'add-rls-to-buckets': 7, + 'add-public-to-buckets': 8, + 'fix-search-function': 9, + 'search-files-search-function': 10, + 'add-trigger-to-auto-update-updated_at-column': 11, + 'add-automatic-avif-detection-flag': 12, + 'add-bucket-custom-limits': 13, + 'use-bytes-for-max-size': 14, + 'add-can-insert-object-function': 15, + 'add-version': 16, + 'drop-owner-foreign-key': 17, + 'add_owner_id_column_deprecate_owner': 18, + 'alter-default-value-objects-id': 19, + 'list-objects-with-delimiter': 20, + 's3-multipart-uploads': 21, + 's3-multipart-uploads-big-ints': 22, + 'optimize-search-function': 23, + 'operation-function': 24, + 'custom-metadata': 25, + 'objects-prefixes': 26, + 'search-v2': 27, + 'object-bucket-name-sorting': 28, + 'create-prefixes': 29, + 'update-object-levels': 30, + 'objects-level-index': 31, + 'backward-compatible-index-on-objects': 32, + 'backward-compatible-index-on-prefixes': 33, + 'optimize-search-function-v1': 34, + 'add-insert-trigger-prefixes': 35, + 'optimise-existing-functions': 36, + 'add-bucket-name-length-trigger': 37, + 'iceberg-catalog-flag-on-buckets': 38, + 'add-search-v2-sort-support': 39, + 'fix-prefix-race-conditions-optimized': 40, + 'add-object-level-update-trigger': 41, + 'rollback-prefix-triggers': 42, + 'fix-object-level': 43, + 'vector-bucket-type': 44, + 'vector-buckets': 45, + 'buckets-objects-grants': 46, + 'iceberg-table-metadata': 47, + 'iceberg-catalog-ids': 48, + 'buckets-objects-grants-postgres': 49, + 'search-v2-optimised': 50, + 'index-backward-compatible-search': 51, + 'drop-not-used-indexes-and-functions': 52, + 'drop-index-lower-name': 53, + 'drop-index-object-level': 54, + 'prevent-direct-deletes': 55, + 'fix-optimized-search-function': 56, + 's3-multipart-uploads-metadata': 57, + } + \ No newline at end of file diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts index 8cba19eb6..e2285fefa 100644 --- a/src/storage/database/adapter.ts +++ b/src/storage/database/adapter.ts @@ -195,7 +195,8 @@ export interface Database { version: string, signature: string, owner?: string, - metadata?: Record + userMetadata?: Record, + metadata?: Partial ): Promise findMultipartUpload( diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index 9d27318ba..27cc7c3b8 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -904,7 +904,8 @@ export class StorageKnexDB implements Database { version: string, signature: string, owner?: string, - metadata?: Record + userMetadata?: Record, + metadata?: Partial ) { return this.runQuery('CreateMultipartUpload', async (knex, signal) => { const multipart = await knex @@ -917,7 +918,8 @@ export class StorageKnexDB implements Database { version, upload_signature: signature, owner_id: owner, - user_metadata: metadata, + user_metadata: userMetadata, + metadata: metadata, }) ) .returning('*') diff --git a/src/storage/object.ts b/src/storage/object.ts index 5ec845728..73b17e4e4 100644 --- a/src/storage/object.ts +++ b/src/storage/object.ts @@ -6,7 +6,7 @@ import { getJwtSecret } from '@internal/database' import { ObjectMetadata, StorageBackendAdapter } from './backend' import { Database, FindObjectFilters, SearchObjectOption } from './database' import { mustBeValidKey } from './limits' -import { fileUploadFromRequest, Uploader, UploadRequest } from './uploader' +import { fileUploadFromRequest, Uploader, UploadRequest, CanUploadMetadata } from './uploader' import { getConfig } from '../config' import { ObjectAdminDelete, @@ -341,6 +341,7 @@ export class ObjectStorage { owner, isUpsert: upsert, userMetadata: userMetadata, + metadata: destinationMetadata, }) try { @@ -794,7 +795,11 @@ export class ObjectStorage { url: string, expiresIn: number, owner?: string, - options?: { upsert?: boolean; userMetadata?: Record } + options?: { + upsert?: boolean + userMetadata?: Record + metadata?: CanUploadMetadata + } ) { // check if user has INSERT permissions await this.uploader.canUpload({ @@ -803,6 +808,7 @@ export class ObjectStorage { owner, isUpsert: options?.upsert ?? false, userMetadata: options?.userMetadata, + metadata: options?.metadata, }) const { urlSigningKey } = await getJwtSecret(this.db.tenantId) diff --git a/src/storage/protocols/s3/s3-handler.ts b/src/storage/protocols/s3/s3-handler.ts index 9ae4dba53..d63ff7e53 100644 --- a/src/storage/protocols/s3/s3-handler.ts +++ b/src/storage/protocols/s3/s3-handler.ts @@ -414,6 +414,9 @@ export class S3ProtocolHandler { isUpsert: true, owner: this.owner, userMetadata: command.Metadata, + metadata: { + mimetype: command.ContentType, + }, }) const uploadId = await this.storage.backend.createMultiPartUpload( @@ -442,7 +445,8 @@ export class S3ProtocolHandler { version, signature, this.owner, - command.Metadata + command.Metadata, + { mimetype: command.ContentType } ) return { @@ -473,7 +477,7 @@ export class S3ProtocolHandler { const multiPartUpload = await this.storage.db .asSuperUser() - .findMultipartUpload(UploadId, 'id,version,user_metadata') + .findMultipartUpload(UploadId, 'id,version,user_metadata,metadata') await uploader.canUpload({ bucketId: Bucket as string, @@ -481,6 +485,7 @@ export class S3ProtocolHandler { isUpsert: true, owner: this.owner, userMetadata: multiPartUpload.user_metadata || undefined, + metadata: multiPartUpload.metadata || undefined, }) const parts = command.MultipartUpload?.Parts || [] @@ -589,6 +594,7 @@ export class S3ProtocolHandler { owner: this.owner, isUpsert: true, userMetadata: multipart.user_metadata || undefined, + metadata: multipart.metadata || undefined, }) if (signal?.aborted) { @@ -739,7 +745,7 @@ export class S3ProtocolHandler { const multipart = await this.storage.db .asSuperUser() - .findMultipartUpload(UploadId, 'id,version,user_metadata') + .findMultipartUpload(UploadId, 'id,version,user_metadata,metadata') const uploader = new Uploader(this.storage.backend, this.storage.db, this.storage.location) await uploader.canUpload({ @@ -748,6 +754,7 @@ export class S3ProtocolHandler { owner: this.owner, isUpsert: true, userMetadata: multipart.user_metadata || undefined, + metadata: multipart.metadata || undefined, }) await this.storage.backend.abortMultipartUpload( @@ -1257,6 +1264,7 @@ export class S3ProtocolHandler { owner: this.owner, isUpsert: true, userMetadata: multipart.user_metadata || undefined, + metadata: multipart.metadata || undefined, }) const uploadPart = await this.storage.backend.uploadPartCopy( @@ -1330,7 +1338,7 @@ export class S3ProtocolHandler { return this.storage.db.asSuperUser().withTransaction(async (db) => { const multipart = await db.findMultipartUpload( uploadId, - 'in_progress_size,version,upload_signature,user_metadata', + 'in_progress_size,version,upload_signature,user_metadata,metadata', { forUpdate: true, } diff --git a/src/storage/schemas/multipart.ts b/src/storage/schemas/multipart.ts index 1ca6eb296..3a1849ac8 100644 --- a/src/storage/schemas/multipart.ts +++ b/src/storage/schemas/multipart.ts @@ -15,6 +15,9 @@ export const multipartUploadSchema = { user_metadata: { anyOf: [{ type: 'object', additionalProperties: true }, { type: 'null' }], }, + metadata: { + anyOf: [{ type: 'object', additionalProperties: true }, { type: 'null' }], + }, }, required: [ 'id', diff --git a/src/storage/uploader.ts b/src/storage/uploader.ts index 7a862e290..eb3c7c147 100644 --- a/src/storage/uploader.ts +++ b/src/storage/uploader.ts @@ -22,6 +22,7 @@ interface FileUpload { cacheControl: string isTruncated: () => boolean xRobotsTag?: string + contentLength?: number } export interface UploadRequest { @@ -35,6 +36,18 @@ export interface UploadRequest { signal?: AbortSignal } +export type CanUploadMetadata = Partial> & + Record + +export interface CanUploadOptions { + bucketId: string + objectName: string + owner: string | undefined + isUpsert: boolean | undefined + userMetadata: Record | undefined + metadata: CanUploadMetadata | undefined +} + const MAX_CUSTOM_METADATA_SIZE = 1024 * 1024 /** @@ -48,9 +61,7 @@ export class Uploader { private readonly location: StorageObjectLocator ) {} - async canUpload( - options: Pick - ) { + async canUpload(options: CanUploadOptions) { const shouldCreateObject = !options.isUpsert if (shouldCreateObject) { @@ -60,6 +71,7 @@ export class Uploader { name: options.objectName, version: '1', owner: options.owner, + metadata: options.metadata, user_metadata: options.userMetadata, }) }) @@ -70,6 +82,7 @@ export class Uploader { name: options.objectName, version: '1', owner: options.owner, + metadata: options.metadata, user_metadata: options.userMetadata, }) }) @@ -81,7 +94,7 @@ export class Uploader { * We check RLS policies before proceeding * @param options */ - async prepareUpload(options: Omit) { + async prepareUpload(options: CanUploadOptions & { uploadType?: string }) { await this.canUpload(options) fileUploadStarted.add(1, { uploadType: options.uploadType, @@ -98,7 +111,15 @@ export class Uploader { * @param options */ async upload(request: UploadRequest) { - const version = await this.prepareUpload(request) + const version = await this.prepareUpload({ + bucketId: request.bucketId, + objectName: request.objectName, + owner: request.owner, + isUpsert: request.isUpsert, + userMetadata: request.userMetadata, + metadata: { mimetype: request.file.mimeType, contentLength: request.file.contentLength }, + uploadType: request.uploadType, + }) try { const file = request.file @@ -315,7 +336,12 @@ export async function fileUploadFromRequest( objectName: string } ): Promise< - FileUpload & { maxFileSize: number; userMetadata: Record | undefined } + FileUpload & { + mimeType: string + maxFileSize: number + userMetadata: Record | undefined + contentLength: number | undefined + } > { const contentType = request.headers['content-type'] const xRobotsTag = request.headers['x-robots-tag'] as string | undefined @@ -415,6 +441,10 @@ export async function fileUploadFromRequest( throw ERRORS.NoContentProvided(new Error('Request stream closed before upload could begin')) } + const contentLength = request.headers['content-length'] + ? Number(request.headers['content-length']) + : undefined + return { body, mimeType, @@ -423,6 +453,7 @@ export async function fileUploadFromRequest( userMetadata, maxFileSize, xRobotsTag, + contentLength, } } diff --git a/src/test/rls.test.ts b/src/test/rls.test.ts index f2bd5a233..e660f0e5c 100644 --- a/src/test/rls.test.ts +++ b/src/test/rls.test.ts @@ -59,6 +59,8 @@ interface TestCaseAssert { role?: string policies?: string[] userMetadata?: Record + mimeType?: string + contentLength?: number status: number error?: string } @@ -238,6 +240,8 @@ describe('RLS policies', () => { objectName: objectName, jwt: assert.role === 'service' ? await serviceKeyAsync : jwt, userMetadata: assert.userMetadata, + mimeType: assert.mimeType, + contentLength: assert.contentLength, }) console.log( @@ -301,15 +305,17 @@ async function runOperation( jwt: string objectName: string userMetadata?: Record + mimeType?: string + contentLength?: number } ) { - const { jwt, bucket, objectName, userMetadata } = options + const { jwt, bucket, objectName, userMetadata, mimeType, contentLength } = options switch (operation) { case 'upload': - return uploadFile(bucket, objectName, jwt, false, userMetadata) + return uploadFile(bucket, objectName, jwt, false, userMetadata, mimeType, contentLength) case 'upload.upsert': - return uploadFile(bucket, objectName, jwt, true, userMetadata) + return uploadFile(bucket, objectName, jwt, true, userMetadata, mimeType, contentLength) case 'bucket.list': return appInstance.inject({ method: 'GET', @@ -466,7 +472,9 @@ async function uploadFile( fileName: string, jwt: string, upsert?: boolean, - userMetadata?: Record + userMetadata?: Record, + mimeType?: string, + contentLength?: number ) { const testFile = fs.createReadStream(path.resolve(__dirname, 'assets', 'sadcat.jpg')) const form = new FormData() @@ -476,9 +484,14 @@ async function uploadFile( form.append('metadata', JSON.stringify(userMetadata)) } + if (mimeType) { + form.append('contentType', mimeType) + } + const headers = Object.assign({}, form.getHeaders(), { authorization: `Bearer ${jwt}`, ...(upsert ? { 'x-upsert': 'true' } : {}), + ...(contentLength ? { 'content-length': contentLength.toString() } : {}), }) return appInstance.inject({ diff --git a/src/test/rls_tests.yaml b/src/test/rls_tests.yaml index d4952e63d..27435090e 100644 --- a/src/test/rls_tests.yaml +++ b/src/test/rls_tests.yaml @@ -53,6 +53,18 @@ policies: permissions: ['insert'] content: "WITH CHECK(user_metadata->>'department' = 'engineering')" + - name: insert_only_images + tables: ['storage.objects'] + roles: ['authenticated'] + permissions: ['insert'] + content: "WITH CHECK(metadata->>'mimetype' LIKE 'image/%')" + + - name: insert_max_size_limit + tables: ['storage.objects'] + roles: ['authenticated'] + permissions: ['insert'] + content: "WITH CHECK((metadata->>'contentLength')::int <= 100000)" + tests: - description: 'Will only able to read objects' policies: @@ -495,3 +507,32 @@ tests: - operation: upload status: 400 error: 'new row violates row-level security policy' + + - description: 'Will only upload image files based on mimetype' + policies: + - insert_only_images + asserts: + - operation: upload + objectName: 'test_image.jpg' + mimeType: 'image/jpeg' + status: 200 + + - operation: upload + objectName: 'test_file.txt' + mimeType: 'text/plain' + status: 400 + error: 'new row violates row-level security policy' + + - description: 'Will only upload files under size limit based on contentLength' + policies: + - insert_max_size_limit + asserts: + - operation: upload + objectName: 'small_file.jpg' + status: 200 + + - operation: upload + objectName: 'large_file.jpg' + contentLength: 200000 + status: 400 + error: 'new row violates row-level security policy' From e08151ae23ed41f6aef06df13c788592d8ca8ec5 Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Mon, 9 Feb 2026 09:43:43 -0600 Subject: [PATCH 4/8] refactor: simplify userMetadata and contentLength assignment in upload URL signing --- src/http/routes/object/getSignedUploadURL.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http/routes/object/getSignedUploadURL.ts b/src/http/routes/object/getSignedUploadURL.ts index 02159ddab..6016cb899 100644 --- a/src/http/routes/object/getSignedUploadURL.ts +++ b/src/http/routes/object/getSignedUploadURL.ts @@ -88,10 +88,10 @@ export default async function routes(fastify: FastifyInstance) { .from(bucketName) .signUploadObjectUrl(objectName, urlPath as string, uploadSignedUrlExpirationTime, owner, { upsert: request.headers['x-upsert'] === 'true', - userMetadata: userMetadata, + userMetadata, metadata: { mimetype: contentType, - contentLength: contentLength, + contentLength, }, }) From 3df43c6d822e12974cc4cda649ba264228dccdda Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Mon, 9 Feb 2026 10:55:20 -0600 Subject: [PATCH 5/8] fix: migration file number --- ...ploads-metadata.sql => 0057-s3-multipart-uploads-metadata.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename migrations/tenant/{0056-s3-multipart-uploads-metadata.sql => 0057-s3-multipart-uploads-metadata.sql} (100%) diff --git a/migrations/tenant/0056-s3-multipart-uploads-metadata.sql b/migrations/tenant/0057-s3-multipart-uploads-metadata.sql similarity index 100% rename from migrations/tenant/0056-s3-multipart-uploads-metadata.sql rename to migrations/tenant/0057-s3-multipart-uploads-metadata.sql From e9c011d96970ac38134b65a56c4da12010ff72ad Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Mon, 9 Feb 2026 11:41:03 -0600 Subject: [PATCH 6/8] feat: add contentLength to S3 object creation parameters --- src/storage/protocols/s3/s3-handler.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storage/protocols/s3/s3-handler.ts b/src/storage/protocols/s3/s3-handler.ts index d63ff7e53..4e347b8d8 100644 --- a/src/storage/protocols/s3/s3-handler.ts +++ b/src/storage/protocols/s3/s3-handler.ts @@ -705,6 +705,7 @@ export class S3ProtocolHandler { cacheControl: command.CacheControl!, mimeType: command.ContentType!, isTruncated: options.isTruncated, + contentLength: command.ContentLength, }, objectName: command.Key as string, userMetadata: command.Metadata, From 37f88172e832cd0cd9291e9672b51a4953e40315 Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Mon, 9 Feb 2026 12:01:36 -0600 Subject: [PATCH 7/8] fix: add migration guard for metadata column on s3_multipart_uploads --- src/storage/database/knex.ts | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index 27cc7c3b8..d4214738b 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -908,20 +908,26 @@ export class StorageKnexDB implements Database { metadata?: Partial ) { return this.runQuery('CreateMultipartUpload', async (knex, signal) => { + const data: Record = { + id: uploadId, + bucket_id: bucketId, + key: objectName, + version, + upload_signature: signature, + owner_id: owner, + user_metadata: userMetadata, + } + + if ( + !this.latestMigration || + DBMigration[this.latestMigration] >= DBMigration['s3-multipart-uploads-metadata'] + ) { + data.metadata = metadata + } + const multipart = await knex .table('s3_multipart_uploads') - .insert( - this.normalizeColumns({ - id: uploadId, - bucket_id: bucketId, - key: objectName, - version, - upload_signature: signature, - owner_id: owner, - user_metadata: userMetadata, - metadata: metadata, - }) - ) + .insert(this.normalizeColumns(data)) .returning('*') .abortOnSignal(signal) From 8c316f68989f05020911e1211f6c22119eb50a93 Mon Sep 17 00:00:00 2001 From: Tyler Hillery Date: Tue, 10 Feb 2026 14:25:01 -0600 Subject: [PATCH 8/8] fix: handle tus uploads differently from first upload vs rest --- src/http/routes/tus/index.ts | 2 +- src/http/routes/tus/lifecycle.ts | 57 ++++++---- src/internal/database/migrations/types.ts | 122 +++++++++++----------- src/test/rls.test.ts | 73 ++++++++++++- src/test/rls_tests.yaml | 31 ++++++ 5 files changed, 201 insertions(+), 84 deletions(-) diff --git a/src/http/routes/tus/index.ts b/src/http/routes/tus/index.ts index 097c5a034..8700500d1 100644 --- a/src/http/routes/tus/index.ts +++ b/src/http/routes/tus/index.ts @@ -137,7 +137,7 @@ function createTusServer( namingFunction: namingFunction, onUploadCreate: onCreate, onUploadFinish: onUploadFinish, - onIncomingRequest: onIncomingRequest, + onIncomingRequest: (req, id) => onIncomingRequest(req, id, datastore), generateUrl: generateUrl, getFileIdFromRequest: getFileIdFromRequest, onResponseError: onResponseError, diff --git a/src/http/routes/tus/lifecycle.ts b/src/http/routes/tus/lifecycle.ts index 2c57a51d7..deee5bd5f 100644 --- a/src/http/routes/tus/lifecycle.ts +++ b/src/http/routes/tus/lifecycle.ts @@ -1,6 +1,6 @@ import http from 'http' import { BaseLogger } from 'pino' -import { Metadata, Upload } from '@tus/server' +import { DataStore, Metadata, Upload } from '@tus/server' import { randomUUID } from 'crypto' import { TenantConnection } from '@internal/database' import { ERRORS, isRenderableError } from '@internal/errors' @@ -44,7 +44,7 @@ export type MultiPartRequest = http.IncomingMessage & { /** * Runs on every TUS incoming request */ -export async function onIncomingRequest(rawReq: Request, id: string) { +export async function onIncomingRequest(rawReq: Request, id: string, datastore: DataStore) { const req = getNodeRequest(rawReq) const res = rawReq.node?.res as http.ServerResponse @@ -63,20 +63,6 @@ export async function onIncomingRequest(rawReq: Request, id: string) { req.upload.resources = [`${uploadID.bucket}/${uploadID.objectName}`] - let customMd: Record | undefined = undefined - const uploadMetadataHeader = req.headers['upload-metadata'] - - if (uploadMetadataHeader && typeof uploadMetadataHeader === 'string') { - try { - const parsedMetadata = Metadata.parse(uploadMetadataHeader) - if (parsedMetadata?.metadata) { - customMd = JSON.parse(parsedMetadata.metadata) - } - } catch (e) { - req.log.warn({ error: e }, 'Failed to parse user metadata') - } - } - // Handle signed url requests if (req.url?.startsWith(`/upload/resumable/sign`)) { const signature = req.headers['x-signature'] @@ -106,9 +92,42 @@ export async function onIncomingRequest(rawReq: Request, id: string) { req.upload.storage.location ) - const uploadLength = req.headers['upload-length'] - const contentLength = uploadLength ? Number(uploadLength) : undefined - const contentType = req.headers['content-type'] + let contentType: string | undefined + let contentLength: number | undefined + let rawMetadata: string | null | undefined + + if (req.method === 'POST') { + const uploadMetadataHeader = req.headers['upload-metadata'] + if (uploadMetadataHeader && typeof uploadMetadataHeader === 'string') { + try { + const parsedMetadata = Metadata.parse(uploadMetadataHeader) + contentType = parsedMetadata?.contentType ?? undefined + rawMetadata = parsedMetadata?.metadata + } catch (e) { + req.log.warn({ error: e }, 'Failed to parse upload metadata') + throw ERRORS.InvalidParameter('upload-metadata', { + error: e as Error, + message: 'Invalid Upload-Metadata header', + }) + } + } + const uploadLength = req.headers['upload-length'] + contentLength = uploadLength ? Number(uploadLength) : undefined + } else { + const upload = await datastore.getUpload(id) + contentType = upload.metadata?.contentType ?? undefined + contentLength = upload.size ?? undefined + rawMetadata = upload.metadata?.metadata + } + + let customMd: Record | undefined + if (rawMetadata) { + try { + customMd = JSON.parse(rawMetadata) + } catch (e) { + req.log.warn({ error: e }, 'Failed to parse user metadata') + } + } await uploader.canUpload({ owner: req.upload.owner, diff --git a/src/internal/database/migrations/types.ts b/src/internal/database/migrations/types.ts index bebd77633..1abbd291c 100644 --- a/src/internal/database/migrations/types.ts +++ b/src/internal/database/migrations/types.ts @@ -1,62 +1,60 @@ - - export const DBMigration = { - 'create-migrations-table': 0, - 'initialmigration': 1, - 'storage-schema': 2, - 'pathtoken-column': 3, - 'add-migrations-rls': 4, - 'add-size-functions': 5, - 'change-column-name-in-get-size': 6, - 'add-rls-to-buckets': 7, - 'add-public-to-buckets': 8, - 'fix-search-function': 9, - 'search-files-search-function': 10, - 'add-trigger-to-auto-update-updated_at-column': 11, - 'add-automatic-avif-detection-flag': 12, - 'add-bucket-custom-limits': 13, - 'use-bytes-for-max-size': 14, - 'add-can-insert-object-function': 15, - 'add-version': 16, - 'drop-owner-foreign-key': 17, - 'add_owner_id_column_deprecate_owner': 18, - 'alter-default-value-objects-id': 19, - 'list-objects-with-delimiter': 20, - 's3-multipart-uploads': 21, - 's3-multipart-uploads-big-ints': 22, - 'optimize-search-function': 23, - 'operation-function': 24, - 'custom-metadata': 25, - 'objects-prefixes': 26, - 'search-v2': 27, - 'object-bucket-name-sorting': 28, - 'create-prefixes': 29, - 'update-object-levels': 30, - 'objects-level-index': 31, - 'backward-compatible-index-on-objects': 32, - 'backward-compatible-index-on-prefixes': 33, - 'optimize-search-function-v1': 34, - 'add-insert-trigger-prefixes': 35, - 'optimise-existing-functions': 36, - 'add-bucket-name-length-trigger': 37, - 'iceberg-catalog-flag-on-buckets': 38, - 'add-search-v2-sort-support': 39, - 'fix-prefix-race-conditions-optimized': 40, - 'add-object-level-update-trigger': 41, - 'rollback-prefix-triggers': 42, - 'fix-object-level': 43, - 'vector-bucket-type': 44, - 'vector-buckets': 45, - 'buckets-objects-grants': 46, - 'iceberg-table-metadata': 47, - 'iceberg-catalog-ids': 48, - 'buckets-objects-grants-postgres': 49, - 'search-v2-optimised': 50, - 'index-backward-compatible-search': 51, - 'drop-not-used-indexes-and-functions': 52, - 'drop-index-lower-name': 53, - 'drop-index-object-level': 54, - 'prevent-direct-deletes': 55, - 'fix-optimized-search-function': 56, - 's3-multipart-uploads-metadata': 57, - } - \ No newline at end of file +export const DBMigration = { + 'create-migrations-table': 0, + initialmigration: 1, + 'storage-schema': 2, + 'pathtoken-column': 3, + 'add-migrations-rls': 4, + 'add-size-functions': 5, + 'change-column-name-in-get-size': 6, + 'add-rls-to-buckets': 7, + 'add-public-to-buckets': 8, + 'fix-search-function': 9, + 'search-files-search-function': 10, + 'add-trigger-to-auto-update-updated_at-column': 11, + 'add-automatic-avif-detection-flag': 12, + 'add-bucket-custom-limits': 13, + 'use-bytes-for-max-size': 14, + 'add-can-insert-object-function': 15, + 'add-version': 16, + 'drop-owner-foreign-key': 17, + add_owner_id_column_deprecate_owner: 18, + 'alter-default-value-objects-id': 19, + 'list-objects-with-delimiter': 20, + 's3-multipart-uploads': 21, + 's3-multipart-uploads-big-ints': 22, + 'optimize-search-function': 23, + 'operation-function': 24, + 'custom-metadata': 25, + 'objects-prefixes': 26, + 'search-v2': 27, + 'object-bucket-name-sorting': 28, + 'create-prefixes': 29, + 'update-object-levels': 30, + 'objects-level-index': 31, + 'backward-compatible-index-on-objects': 32, + 'backward-compatible-index-on-prefixes': 33, + 'optimize-search-function-v1': 34, + 'add-insert-trigger-prefixes': 35, + 'optimise-existing-functions': 36, + 'add-bucket-name-length-trigger': 37, + 'iceberg-catalog-flag-on-buckets': 38, + 'add-search-v2-sort-support': 39, + 'fix-prefix-race-conditions-optimized': 40, + 'add-object-level-update-trigger': 41, + 'rollback-prefix-triggers': 42, + 'fix-object-level': 43, + 'vector-bucket-type': 44, + 'vector-buckets': 45, + 'buckets-objects-grants': 46, + 'iceberg-table-metadata': 47, + 'iceberg-catalog-ids': 48, + 'buckets-objects-grants-postgres': 49, + 'search-v2-optimised': 50, + 'index-backward-compatible-search': 51, + 'drop-not-used-indexes-and-functions': 52, + 'drop-index-lower-name': 53, + 'drop-index-object-level': 54, + 'prevent-direct-deletes': 55, + 'fix-optimized-search-function': 56, + 's3-multipart-uploads-metadata': 57, +} diff --git a/src/test/rls.test.ts b/src/test/rls.test.ts index e660f0e5c..3436998f5 100644 --- a/src/test/rls.test.ts +++ b/src/test/rls.test.ts @@ -6,6 +6,8 @@ import FormData from 'form-data' import yaml from 'js-yaml' import Mustache from 'mustache' import { CreateBucketCommand, S3Client } from '@aws-sdk/client-s3' +import * as tus from 'tus-js-client' +import { DetailedError } from 'tus-js-client' import { StorageKnexDB } from '@storage/database' import { createStorageBackend } from '@storage/backend' @@ -42,6 +44,7 @@ interface TestCaseAssert { operation: | 'upload' | 'upload.upsert' + | 'upload.tus' | 'bucket.create' | 'bucket.get' | 'bucket.list' @@ -260,8 +263,7 @@ describe('RLS policies', () => { } if (assert.error) { - const body = await response.json() - + const body = response.json() expect(body.message).toBe(assert.error) } } finally { @@ -316,6 +318,8 @@ async function runOperation( return uploadFile(bucket, objectName, jwt, false, userMetadata, mimeType, contentLength) case 'upload.upsert': return uploadFile(bucket, objectName, jwt, true, userMetadata, mimeType, contentLength) + case 'upload.tus': + return tusUploadFile(bucket, objectName, jwt, userMetadata, mimeType, contentLength) case 'bucket.list': return appInstance.inject({ method: 'GET', @@ -501,3 +505,68 @@ async function uploadFile( payload: form, }) } + +async function tusUploadFile( + bucket: string, + objectName: string, + jwt: string, + userMetadata?: Record, + mimeType?: string, + contentLength?: number +) { + if (!appInstance.server.listening) { + await appInstance.listen({ port: 0 }) + } + + const addressInfo = appInstance.server.address() + if (!addressInfo || typeof addressInfo === 'string') { + throw new Error('Unable to resolve local server address') + } + + const localServerAddress = `http://127.0.0.1:${addressInfo.port}` + + const file = fs.createReadStream(path.resolve(__dirname, 'assets', 'sadcat.jpg')) + + let statusCode = 200 + let message = '' + + try { + await new Promise((resolve, reject) => { + const upload = new tus.Upload(file, { + endpoint: `${localServerAddress}/upload/resumable`, + uploadSize: contentLength || undefined, + onShouldRetry: () => false, + uploadDataDuringCreation: false, + headers: { + authorization: `Bearer ${jwt}`, + }, + metadata: { + bucketName: bucket, + objectName: objectName, + contentType: mimeType || 'application/octet-stream', + cacheControl: '3600', + ...(userMetadata ? { metadata: JSON.stringify(userMetadata) } : {}), + }, + onError: function (error) { + console.log('Failed because: ' + error) + reject(error) + }, + onSuccess: () => { + resolve(true) + }, + }) + + upload.start() + }) + } catch (e) { + if (e instanceof DetailedError) { + statusCode = e.originalResponse.getStatus() + message = e.originalResponse.getBody() + } else { + throw e + } + } + + const body = message ? { message } : {} + return { statusCode, body: JSON.stringify(body), json: () => body } +} diff --git a/src/test/rls_tests.yaml b/src/test/rls_tests.yaml index 27435090e..ae8820216 100644 --- a/src/test/rls_tests.yaml +++ b/src/test/rls_tests.yaml @@ -508,6 +508,16 @@ tests: status: 400 error: 'new row violates row-level security policy' + - operation: upload.tus + objectName: 'test_file_tus.jpg' + userMetadata: + department: 'engineering' + status: 200 + + - operation: upload.tus + status: 403 + error: 'new row violates row-level security policy' + - description: 'Will only upload image files based on mimetype' policies: - insert_only_images @@ -523,6 +533,17 @@ tests: status: 400 error: 'new row violates row-level security policy' + - operation: upload.tus + objectName: 'test_image_tus.jpg' + mimeType: 'image/jpeg' + status: 200 + + - operation: upload.tus + objectName: 'test_file_tus.txt' + mimeType: 'text/plain' + status: 403 + error: 'new row violates row-level security policy' + - description: 'Will only upload files under size limit based on contentLength' policies: - insert_max_size_limit @@ -536,3 +557,13 @@ tests: contentLength: 200000 status: 400 error: 'new row violates row-level security policy' + + - operation: upload.tus + objectName: 'small_file_tus.jpg' + status: 200 + + - operation: upload.tus + objectName: 'large_file_tus.jpg' + contentLength: 200000 + status: 403 + error: 'new row violates row-level security policy'