From aad4da0445737dd4e30bfe500fe7b2569f57b3dd Mon Sep 17 00:00:00 2001 From: Brage Sekse Aarset Date: Wed, 31 Jan 2024 15:23:13 +0200 Subject: [PATCH 1/7] feat: store matrix data in storage bucket --- resources/google/gke.ts | 3 + .../components/standard-deployment.ts | 19 ++++- resources/kubernetes/deployments/matrix.ts | 78 ++++++------------- resources/kubernetes/matrix/fuse-csi.ts | 21 +++++ resources/kubernetes/matrix/google.ts | 28 +++++++ resources/kubernetes/matrix/k8s.ts | 28 +++++++ 6 files changed, 121 insertions(+), 56 deletions(-) create mode 100644 resources/kubernetes/matrix/fuse-csi.ts create mode 100644 resources/kubernetes/matrix/google.ts create mode 100644 resources/kubernetes/matrix/k8s.ts diff --git a/resources/google/gke.ts b/resources/google/gke.ts index 7841c47d..b1fdf56f 100644 --- a/resources/google/gke.ts +++ b/resources/google/gke.ts @@ -9,6 +9,9 @@ export const cluster = new google.container.v1.Cluster( releaseChannel: { channel: 'REGULAR' }, location: region, autopilot: { enabled: true }, + addonsConfig: { + gcsFuseCsiDriverConfig: { enabled: true }, + }, }, { provider: mainProvider, protect: true }, ); diff --git a/resources/kubernetes/components/standard-deployment.ts b/resources/kubernetes/components/standard-deployment.ts index a581a1c1..f1e4d96b 100644 --- a/resources/kubernetes/components/standard-deployment.ts +++ b/resources/kubernetes/components/standard-deployment.ts @@ -148,6 +148,20 @@ export interface StandardDeploymentArgs */ securityContext?: pulumi.Input; + /** + * Annotations is an unstructured key value map stored with a resource that may be set by external + * tools to store and retrieve arbitrary metadata. + * They are not queryable and should be preserved when modifying objects. + * More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + */ + annotations?: Record>; + + /** + * ServiceAccountName is the name of the ServiceAccount to use to run this pod. + * More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + */ + serviceAccountName?: pulumi.Input; + /** * Arguments to the entrypoint. The container image's CMD is used if this is not provided. * Variable references $(VAR_NAME) are expanded using the container's environment. @@ -229,6 +243,8 @@ export class StandardDeployment extends pulumi.ComponentResource { args: entrypointArgs = undefined, volumeMounts = [], securityContext = undefined, + annotations = {}, + serviceAccountName = undefined, } = args; const publicPort = ports.find(p => p.name === 'public'); @@ -318,7 +334,6 @@ export class StandardDeployment extends pulumi.ComponentResource { } : undefined, }; - this.deployment = new k8s.apps.v1.Deployment( name, { @@ -326,6 +341,7 @@ export class StandardDeployment extends pulumi.ComponentResource { name, annotations: { 'pulumi.com/skipAwait': 'true', + ...annotations, }, }, spec: { @@ -343,6 +359,7 @@ export class StandardDeployment extends pulumi.ComponentResource { }, spec: { securityContext, + serviceAccountName, volumes, initContainers: pulumi.output(initContainers).apply(ic => ic.map(initContainer => ({ diff --git a/resources/kubernetes/deployments/matrix.ts b/resources/kubernetes/deployments/matrix.ts index 1fb19e6b..aae30e42 100644 --- a/resources/kubernetes/deployments/matrix.ts +++ b/resources/kubernetes/deployments/matrix.ts @@ -3,6 +3,8 @@ import * as pulumi from '@pulumi/pulumi'; import * as yaml from 'js-yaml'; import { StandardDatabase } from '../components/standard-database'; import { StandardDeployment } from '../components/standard-deployment'; +import { matrixDataBucket } from '../matrix/google'; +import { matrixK8sServiceAccount } from '../matrix/k8s'; import { provider } from '../provider'; const config = new pulumi.Config('matrix'); @@ -21,45 +23,7 @@ export const synapseDatabase = new StandardDatabase( const registrationSecret = config.requireSecret('registration-secret'); -const mediaVolume = new k8s.core.v1.PersistentVolumeClaim( - 'matrix-media-volume', - { - metadata: { - name: 'matrix-media-volume', - }, - spec: { - accessModes: ['ReadWriteOnce'], - resources: { - requests: { - storage: '1Gi', - }, - }, - storageClassName: 'standard', - }, - }, - { provider }, -); - -const dataVolume = new k8s.core.v1.PersistentVolumeClaim( - 'matrix-data-volume', - { - metadata: { - name: 'matrix-data-volume', - }, - spec: { - accessModes: ['ReadWriteOnce'], - resources: { - requests: { - storage: '1Gi', - }, - }, - storageClassName: 'standard', - }, - }, - { provider }, -); - -const secretVolume = new k8s.core.v1.Secret( +const secret = new k8s.core.v1.Secret( 'matrix-registration-secret', { metadata: { @@ -75,7 +39,9 @@ const secretVolume = new k8s.core.v1.Secret( }, { provider }, ); + const host = config.require('host'); + export const homeserverConfig = new k8s.core.v1.ConfigMap( 'matrix-homeserver-config', { @@ -93,7 +59,7 @@ export const homeserverConfig = new k8s.core.v1.ConfigMap( enable_registration_captcha: true, registration_requires_token: true, report_stats: true, - media_store_path: '/synapse/media_store', + media_store_path: '/synapse/data/media_store', pid_file: '/synapse/data/homeserver.pid', signing_key_path: '/synapse/data/bjerk.io.signing.key', database: { @@ -124,6 +90,13 @@ export const synapseDeployment = new StandardDeployment( image: config.require('synapse-image'), tag: config.require('synapse-tag'), host, + annotations: { + 'gke-gcsfuse/volumes': 'true', + 'gke-gcsfuse/cpu-limit': '500m', + 'gke-gcsfuse/memory-limit': '1Gi', + 'gke-gcsfuse/ephemeral-storage-limit': '50Gi', + }, + serviceAccountName: matrixK8sServiceAccount.metadata.name, healthCheckHttpPath: '/health', databaseDetails: synapseDatabase.databaseDetails, volumes: [ @@ -131,7 +104,7 @@ export const synapseDeployment = new StandardDeployment( name: 'secrets', emptyDir: {}, secret: { - secretName: secretVolume.metadata.name, + secretName: secret.metadata.name, }, }, { @@ -142,15 +115,14 @@ export const synapseDeployment = new StandardDeployment( }, }, { - name: 'data', - persistentVolumeClaim: { - claimName: dataVolume.metadata.name, - }, - }, - { - name: 'media', - persistentVolumeClaim: { - claimName: mediaVolume.metadata.name, + name: 'gcs-fuse-csi-ephemeral', + csi: { + driver: 'gcs.csi.infra.gke.io', + readOnly: true, + volumeAttributes: { + bucketName: matrixDataBucket.name, + mountOptions: 'implicit-dirs', + }, }, }, ], @@ -164,13 +136,9 @@ export const synapseDeployment = new StandardDeployment( mountPath: '/config', }, { - name: 'data', + name: 'gcs-fuse-csi-ephemeral', mountPath: '/synapse/data', }, - { - name: 'media', - mountPath: '/synapse/media_store', - }, ], // This is needed to tell synapse to load the secrets and config from these files command: [ diff --git a/resources/kubernetes/matrix/fuse-csi.ts b/resources/kubernetes/matrix/fuse-csi.ts new file mode 100644 index 00000000..f1cce75d --- /dev/null +++ b/resources/kubernetes/matrix/fuse-csi.ts @@ -0,0 +1,21 @@ +import * as gcp from '@pulumi/gcp'; +import * as pulumi from '@pulumi/pulumi'; +import { project } from '../../google/project'; +import { matrixSynapseServiceAccount } from './google'; +import { matrixK8sServiceAccount } from './k8s'; + +const iamServiceAccountIamBinding = new gcp.serviceaccount.IAMBinding( + 'iamServiceAccountIamBinding', + { + serviceAccountId: matrixSynapseServiceAccount.name, + role: 'roles/iam.serviceAccountTokenCreator', + members: [ + // Construct the Kubernetes service account member string + // It should be in the format "serviceAccount:PROJECT_ID.svc.id.goog[K8S_NAMESPACE/K8S_SA_NAME]" + // PROJECT_ID should be replaced by your Google Cloud project ID + // K8S_NAMESPACE should be replaced by your Kubernetes service account namespace + // K8S_SA_NAME should be replaced by your Kubernetes service account name + pulumi.interpolate`serviceAccount:${project.projectId}.svc.id.goog[${matrixK8sServiceAccount.metadata.namespace}/${matrixK8sServiceAccount.metadata.name}]`, + ], + }, +); diff --git a/resources/kubernetes/matrix/google.ts b/resources/kubernetes/matrix/google.ts new file mode 100644 index 00000000..493852e6 --- /dev/null +++ b/resources/kubernetes/matrix/google.ts @@ -0,0 +1,28 @@ +import * as gcp from '@pulumi/gcp'; +import { region } from '../../config'; + +// Create a Google Cloud Storage bucket +export const matrixDataBucket = new gcp.storage.Bucket( + 'matrix-synapse-media-bucket', + { + location: region, + }, +); + +// Create a Google Cloud service account +export const matrixSynapseServiceAccount = new gcp.serviceaccount.Account( + 'matrix-synapse-service-account', + { + accountId: 'matrix-synapse-service-account', + displayName: 'Matrix Service Account', + }, +); + +// Grant the service account the role of Storage Object Viewer on the bucket +const bucketIamMember = new gcp.storage.BucketIAMMember('bucket-iam-member', { + bucket: matrixDataBucket.name, // reference to our created bucket + role: 'roles/storage.objectViewer', + member: matrixSynapseServiceAccount.email.apply( + email => `serviceAccount:${email}`, + ), // dynamically fetch the service account's email +}); diff --git a/resources/kubernetes/matrix/k8s.ts b/resources/kubernetes/matrix/k8s.ts new file mode 100644 index 00000000..1483e05b --- /dev/null +++ b/resources/kubernetes/matrix/k8s.ts @@ -0,0 +1,28 @@ +import * as k8s from '@pulumi/kubernetes'; +import { provider } from '../provider'; +import { matrixSynapseServiceAccount } from './google'; + +export const matrixK8sServiceAccountNamespace = new k8s.core.v1.Namespace( + 'matrix-service-account-namespace', + { + metadata: { + name: 'service-account', + }, + }, + { provider }, +); + +export const matrixK8sServiceAccount = new k8s.core.v1.ServiceAccount( + 'matrix-service-account', + { + metadata: { + name: 'branches', + namespace: matrixK8sServiceAccountNamespace.metadata.name, + annotations: { + 'iam.gke.io/gcp-service-account': + matrixSynapseServiceAccount.email.apply(email => email), + }, + }, + }, + { provider }, +); From b7e34c54a9116a0d25fccadf8eee9e82b902ea99 Mon Sep 17 00:00:00 2001 From: Brage Sekse Aarset Date: Wed, 31 Jan 2024 15:25:05 +0200 Subject: [PATCH 2/7] fix: objectUser permissions --- resources/kubernetes/matrix/google.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/kubernetes/matrix/google.ts b/resources/kubernetes/matrix/google.ts index 493852e6..ed8b9ae0 100644 --- a/resources/kubernetes/matrix/google.ts +++ b/resources/kubernetes/matrix/google.ts @@ -19,9 +19,9 @@ export const matrixSynapseServiceAccount = new gcp.serviceaccount.Account( ); // Grant the service account the role of Storage Object Viewer on the bucket -const bucketIamMember = new gcp.storage.BucketIAMMember('bucket-iam-member', { +new gcp.storage.BucketIAMMember('bucket-iam-member', { bucket: matrixDataBucket.name, // reference to our created bucket - role: 'roles/storage.objectViewer', + role: 'roles/storage.objectUser', member: matrixSynapseServiceAccount.email.apply( email => `serviceAccount:${email}`, ), // dynamically fetch the service account's email From 2b1d58d96262c4c6fd20560a52dabb6fcdf52fa9 Mon Sep 17 00:00:00 2001 From: Brage Sekse Aarset Date: Wed, 31 Jan 2024 15:25:20 +0200 Subject: [PATCH 3/7] uncomment matrix --- resources/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/index.ts b/resources/index.ts index 0ac27f68..3fd8f78a 100644 --- a/resources/index.ts +++ b/resources/index.ts @@ -18,7 +18,7 @@ import './kubernetes/provider'; // Kubernetes Deployments import './kubernetes/deployments/abax-minuba'; import './kubernetes/deployments/abax-procore'; -// import './kubernetes/deployments/matrix'; +import './kubernetes/deployments/matrix'; import './kubernetes/deployments/abax-vwfs'; import './kubernetes/todoist-github/deployment'; import './kubernetes/todoist-github/ingress'; From 809aabab92f9f6cd9148b4a2c5d37b9eb525fd19 Mon Sep 17 00:00:00 2001 From: Brage Sekse Aarset Date: Wed, 31 Jan 2024 15:28:40 +0200 Subject: [PATCH 4/7] fix: add provider --- resources/kubernetes/matrix/google.ts | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/resources/kubernetes/matrix/google.ts b/resources/kubernetes/matrix/google.ts index ed8b9ae0..c72ce6e6 100644 --- a/resources/kubernetes/matrix/google.ts +++ b/resources/kubernetes/matrix/google.ts @@ -1,5 +1,6 @@ import * as gcp from '@pulumi/gcp'; import { region } from '../../config'; +import { mainProvider } from '../../google/project'; // Create a Google Cloud Storage bucket export const matrixDataBucket = new gcp.storage.Bucket( @@ -7,6 +8,7 @@ export const matrixDataBucket = new gcp.storage.Bucket( { location: region, }, + { provider: mainProvider }, ); // Create a Google Cloud service account @@ -16,13 +18,18 @@ export const matrixSynapseServiceAccount = new gcp.serviceaccount.Account( accountId: 'matrix-synapse-service-account', displayName: 'Matrix Service Account', }, + { provider: mainProvider }, ); // Grant the service account the role of Storage Object Viewer on the bucket -new gcp.storage.BucketIAMMember('bucket-iam-member', { - bucket: matrixDataBucket.name, // reference to our created bucket - role: 'roles/storage.objectUser', - member: matrixSynapseServiceAccount.email.apply( - email => `serviceAccount:${email}`, - ), // dynamically fetch the service account's email -}); +new gcp.storage.BucketIAMMember( + 'bucket-iam-member', + { + bucket: matrixDataBucket.name, // reference to our created bucket + role: 'roles/storage.objectUser', + member: matrixSynapseServiceAccount.email.apply( + email => `serviceAccount:${email}`, + ), // dynamically fetch the service account's email + }, + { provider: mainProvider }, +); From 036f531123b5447d1a1bf30a6befb45fc31799d7 Mon Sep 17 00:00:00 2001 From: Brage Sekse Aarset Date: Wed, 31 Jan 2024 15:29:44 +0200 Subject: [PATCH 5/7] fix: use classic provider --- resources/kubernetes/matrix/fuse-csi.ts | 5 +++-- resources/kubernetes/matrix/google.ts | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/resources/kubernetes/matrix/fuse-csi.ts b/resources/kubernetes/matrix/fuse-csi.ts index f1cce75d..0077becf 100644 --- a/resources/kubernetes/matrix/fuse-csi.ts +++ b/resources/kubernetes/matrix/fuse-csi.ts @@ -1,10 +1,10 @@ import * as gcp from '@pulumi/gcp'; import * as pulumi from '@pulumi/pulumi'; -import { project } from '../../google/project'; +import { mainClassicProvider, project } from '../../google/project'; import { matrixSynapseServiceAccount } from './google'; import { matrixK8sServiceAccount } from './k8s'; -const iamServiceAccountIamBinding = new gcp.serviceaccount.IAMBinding( +new gcp.serviceaccount.IAMBinding( 'iamServiceAccountIamBinding', { serviceAccountId: matrixSynapseServiceAccount.name, @@ -18,4 +18,5 @@ const iamServiceAccountIamBinding = new gcp.serviceaccount.IAMBinding( pulumi.interpolate`serviceAccount:${project.projectId}.svc.id.goog[${matrixK8sServiceAccount.metadata.namespace}/${matrixK8sServiceAccount.metadata.name}]`, ], }, + { provider: mainClassicProvider }, ); diff --git a/resources/kubernetes/matrix/google.ts b/resources/kubernetes/matrix/google.ts index c72ce6e6..e706ce8d 100644 --- a/resources/kubernetes/matrix/google.ts +++ b/resources/kubernetes/matrix/google.ts @@ -1,6 +1,6 @@ import * as gcp from '@pulumi/gcp'; import { region } from '../../config'; -import { mainProvider } from '../../google/project'; +import { mainClassicProvider } from '../../google/project'; // Create a Google Cloud Storage bucket export const matrixDataBucket = new gcp.storage.Bucket( @@ -8,7 +8,7 @@ export const matrixDataBucket = new gcp.storage.Bucket( { location: region, }, - { provider: mainProvider }, + { provider: mainClassicProvider }, ); // Create a Google Cloud service account @@ -18,7 +18,7 @@ export const matrixSynapseServiceAccount = new gcp.serviceaccount.Account( accountId: 'matrix-synapse-service-account', displayName: 'Matrix Service Account', }, - { provider: mainProvider }, + { provider: mainClassicProvider }, ); // Grant the service account the role of Storage Object Viewer on the bucket @@ -31,5 +31,5 @@ new gcp.storage.BucketIAMMember( email => `serviceAccount:${email}`, ), // dynamically fetch the service account's email }, - { provider: mainProvider }, + { provider: mainClassicProvider }, ); From 31c484d6b28709cde29c40bb9f09d01bd644f95e Mon Sep 17 00:00:00 2001 From: Brage Sekse Aarset Date: Mon, 5 Feb 2024 08:42:21 +0200 Subject: [PATCH 6/7] fix: remove comment --- resources/kubernetes/matrix/fuse-csi.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/resources/kubernetes/matrix/fuse-csi.ts b/resources/kubernetes/matrix/fuse-csi.ts index 0077becf..01e5b848 100644 --- a/resources/kubernetes/matrix/fuse-csi.ts +++ b/resources/kubernetes/matrix/fuse-csi.ts @@ -10,11 +10,6 @@ new gcp.serviceaccount.IAMBinding( serviceAccountId: matrixSynapseServiceAccount.name, role: 'roles/iam.serviceAccountTokenCreator', members: [ - // Construct the Kubernetes service account member string - // It should be in the format "serviceAccount:PROJECT_ID.svc.id.goog[K8S_NAMESPACE/K8S_SA_NAME]" - // PROJECT_ID should be replaced by your Google Cloud project ID - // K8S_NAMESPACE should be replaced by your Kubernetes service account namespace - // K8S_SA_NAME should be replaced by your Kubernetes service account name pulumi.interpolate`serviceAccount:${project.projectId}.svc.id.goog[${matrixK8sServiceAccount.metadata.namespace}/${matrixK8sServiceAccount.metadata.name}]`, ], }, From 47afefe4628e7c9ad00e66ef23382f1d86c46381 Mon Sep 17 00:00:00 2001 From: Brage Sekse Aarset Date: Mon, 5 Feb 2024 08:53:33 +0200 Subject: [PATCH 7/7] refactor: use googlenative --- resources/kubernetes/matrix/fuse-csi.ts | 10 ++++---- resources/kubernetes/matrix/google.ts | 32 ++++++++++++++----------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/resources/kubernetes/matrix/fuse-csi.ts b/resources/kubernetes/matrix/fuse-csi.ts index 01e5b848..bbb1bf93 100644 --- a/resources/kubernetes/matrix/fuse-csi.ts +++ b/resources/kubernetes/matrix/fuse-csi.ts @@ -1,17 +1,17 @@ -import * as gcp from '@pulumi/gcp'; +import * as google from '@pulumi/google-native'; import * as pulumi from '@pulumi/pulumi'; -import { mainClassicProvider, project } from '../../google/project'; +import { mainProvider, project } from '../../google/project'; import { matrixSynapseServiceAccount } from './google'; import { matrixK8sServiceAccount } from './k8s'; -new gcp.serviceaccount.IAMBinding( +new google.iam.v1.ServiceAccountIamBinding( 'iamServiceAccountIamBinding', { - serviceAccountId: matrixSynapseServiceAccount.name, + name: matrixSynapseServiceAccount.name, role: 'roles/iam.serviceAccountTokenCreator', members: [ pulumi.interpolate`serviceAccount:${project.projectId}.svc.id.goog[${matrixK8sServiceAccount.metadata.namespace}/${matrixK8sServiceAccount.metadata.name}]`, ], }, - { provider: mainClassicProvider }, + { provider: mainProvider }, ); diff --git a/resources/kubernetes/matrix/google.ts b/resources/kubernetes/matrix/google.ts index e706ce8d..db6d9b15 100644 --- a/resources/kubernetes/matrix/google.ts +++ b/resources/kubernetes/matrix/google.ts @@ -1,35 +1,39 @@ -import * as gcp from '@pulumi/gcp'; +import * as google from '@pulumi/google-native'; import { region } from '../../config'; -import { mainClassicProvider } from '../../google/project'; +import { mainProvider } from '../../google/project'; // Create a Google Cloud Storage bucket -export const matrixDataBucket = new gcp.storage.Bucket( +export const matrixDataBucket = new google.storage.v1.Bucket( 'matrix-synapse-media-bucket', { + name: 'matrix-synapse-media-bucket', location: region, }, - { provider: mainClassicProvider }, + { provider: mainProvider }, ); // Create a Google Cloud service account -export const matrixSynapseServiceAccount = new gcp.serviceaccount.Account( +export const matrixSynapseServiceAccount = new google.iam.v1.ServiceAccount( 'matrix-synapse-service-account', { accountId: 'matrix-synapse-service-account', + name: 'matrix-synapse-service-account', displayName: 'Matrix Service Account', }, - { provider: mainClassicProvider }, + { provider: mainProvider }, ); // Grant the service account the role of Storage Object Viewer on the bucket -new gcp.storage.BucketIAMMember( - 'bucket-iam-member', +new google.storage.v1.BucketIamBinding( + 'bucket-iam-binding', { - bucket: matrixDataBucket.name, // reference to our created bucket - role: 'roles/storage.objectUser', - member: matrixSynapseServiceAccount.email.apply( - email => `serviceAccount:${email}`, - ), // dynamically fetch the service account's email + name: matrixDataBucket.name, + role: 'roles/storage.objectViewer', + members: [ + matrixSynapseServiceAccount.email.apply( + email => `serviceAccount:${email}`, + ), + ], }, - { provider: mainClassicProvider }, + { provider: mainProvider }, );