diff --git a/helm/plater/templates/configmap.yaml b/helm/plater/templates/configmap.yaml index 71f5a4692..a7adef12e 100644 --- a/helm/plater/templates/configmap.yaml +++ b/helm/plater/templates/configmap.yaml @@ -14,11 +14,40 @@ data: echo " -- Neo4j is unavailable - sleeping" sleep 3 done - download_data.sh: |- + download_data_neo4j.sh: |- #!/bin/bash set -x wget -O /data/neo4j.dump ${NEO4J_DATA_URL} touch /data/done + + run_memgraph.sh: |- + #!/bin/sh + set -e + + echo "Starting Memgraph..." + /usr/lib/memgraph/memgraph \ + --bolt-port=7687 \ + --bolt-address=0.0.0.0 \ + --log-level=INFO \ + --also-log-to-stderr \ + & + MEMGRAPH_PID=$! + + echo "Waiting for Memgraph to accept connections...${MEMGRAPH_PID}" + until echo "RETURN 1;" | /usr/bin/mgconsole --host localhost --port 7687 >/dev/null 2>&1; do + sleep 1 + done + + if [ -f /data/memgraph.cypher ]; then + echo "Loading Memgraph dump..." + /usr/bin/mgconsole --host localhost --port 7687 < /data/memgraph.cypher + echo "Memgraph dump loaded" + else + echo "No memgraph.cypher found, skipping load" + fi + + wait ${MEMGRAPH_PID} + health_check.sh: |- #!/usr/bin/env bash response=$(wget --server-response "${PUBLIC_URL}/openapi.json" 2>&1 | awk '/^ HTTP/{print $2}') @@ -66,9 +95,9 @@ data: fi seed_db.sh: |- #!/bin/bash - set -x + set -ex FILE=/data/neo4j.dump - DATADIR=/data/databases + NEO4J_DATA=/data until [ -f "$FILE" ]; do echo "$FILE doesn't exist. Please copy to database dump file to $FILE" echo "sleeping..." @@ -77,15 +106,15 @@ data: echo "$FILE found" echo "clearing previous database file if any..." # Make dir if they don't exist - mkdir -p /data/databases - mkdir -p /data/transactions - mkdir -p /data/dbms + mkdir -p ${NEO4J_DATA}/databases + mkdir -p ${NEO4J_DATA}/transactions + mkdir -p ${NEO4J_DATA}/dbms # clear out data from dir if they were present - rm -rf /data/dbms/* - rm -rf /data/databases/* - rm -rf /data/transactions/* + rm -rf ${NEO4J_DATA}/dbms/* + rm -rf ${NEO4J_DATA}/databases/* + rm -rf ${NEO4J_DATA}/transactions/* # load dump file - neo4j-admin database load --from-path=/data/ neo4j + neo4j-admin database load --from-path=/data/ neo4j --overwrite-destination=true --verbose dataset-desc.json: | {{ .Values.datasetDesc | toJson }} openapi-config.yaml: |- diff --git a/helm/plater/templates/deployment.yaml b/helm/plater/templates/deployment.yaml index 9fa88582d..7eeaa8674 100644 --- a/helm/plater/templates/deployment.yaml +++ b/helm/plater/templates/deployment.yaml @@ -34,26 +34,21 @@ spec: - name: {{ include "plater.fullname" . }}-metadata-pvc mountPath: /home/plater/Plater/PLATER/metadata subPath: plater-metadata/ + {{- if and (eq .Values.graph.backend "neo4j") (not .Values.graph.external) }} - name: {{ include "plater.fullname" . }}-init-container image: "{{ .Values.busybox.image.repository }}:{{ .Values.busybox.image.tag }}" command: - './check_neo.sh' env: - {{ if not .Values.externalNeo4j }} - name: NEO4J_HOST value: {{ include "plater.fullname" . }}-neo4j-service - name: NEO4J_HTTP_PORT value: "{{ .Values.app.neo4j.httpPort }}" - {{ else }} - - name: NEO4J_HOST - value: {{ .Values.externalNeo4j.hostName }} - - name: NEO4J_HTTP_PORT - value: "{{ .Values.externalNeo4j.port }}" - {{ end }} volumeMounts: - name: {{ include "plater.fullname" . }}-config-files mountPath: /check_neo.sh subPath: check_neo.sh + {{- end }} containers: - name: {{ include "plater.fullname" . }}-web-container image: "{{ .Values.image.plater.repository }}:{{ .Values.image.plater.tag }}" diff --git a/helm/plater/templates/env-config-map.yaml b/helm/plater/templates/env-config-map.yaml index d5876e413..199549265 100644 --- a/helm/plater/templates/env-config-map.yaml +++ b/helm/plater/templates/env-config-map.yaml @@ -5,19 +5,34 @@ metadata: data: WEB_HOST: "0.0.0.0" WEB_PORT: "{{ .Values.app.port }}" - {{ if not .Values.externalNeo4j }} + GRAPH_DB: "{{ .Values.graph.backend }}" + GRAPH_QUERY_TIMEOUT: "{{ .Values.app.GRAPH_QUERY_TIMEOUT }}" + {{- if eq .Values.graph.backend "neo4j" }} + {{- if .Values.graph.external }} + NEO4J_HOST: {{ .Values.externalNeo4j.hostName }} + NEO4J_HTTP_PORT: "{{ .Values.externalNeo4j.httpPort }}" + NEO4J_BOLT_PORT: "{{ .Values.externalNeo4j.boltPort }}" + NEO4J_PASSWORD: "{{ .Values.externalNeo4j.password }}" + NEO4J_USERNAME: "{{ .Values.externalNeo4j.username }}" + {{- else }} NEO4J_HOST: {{ include "plater.fullname" . }}-neo4j-service NEO4J_HTTP_PORT: "{{ .Values.app.neo4j.httpPort }}" NEO4J_BOLT_PORT: "{{ .Values.app.neo4j.boltPort }}" NEO4J_PASSWORD: "{{ .Values.app.neo4j.password }}" NEO4J_USERNAME: "{{ .Values.app.neo4j.username }}" - {{ else }} - NEO4J_HOST: {{ .Values.externalNeo4j.hostName }} - NEO4J_HTTP_PORT: "{{ .Values.externalNeo4j.port }}" - NEO4J_PASSWORD: "{{ .Values.externalNeo4j.password }}" - NEO4J_USERNAME: "{{ .Values.externalNeo4j.username }}" - {{ end }} - NEO4J_QUERY_TIMEOUT: "{{ .Values.app.Neo4jQueryTimeout }}" + {{- end }} + {{- end }} + + {{- if eq .Values.graph.backend "memgraph" }} + {{- if .Values.graph.external }} + MEMGRAPH_HOST: {{ .Values.externalMemgraph.hostName | quote }} + MEMGRAPH_BOLT_PORT: "{{ .Values.externalMemgraph.boltPort }}" + {{- else }} + MEMGRAPH_HOST: {{ include "plater.fullname" . }}-memgraph-service + MEMGRAPH_BOLT_PORT: "{{ .Values.app.memgraph.boltPort }}" + {{- end }} + {{- end }} + PLATER_SERVICE_ADDRESS: "{{ include "plater.fullname" . }}-plater-service" AUTOMAT_HOST: {{ .Values.app.automatAddress }} PLATER_TITLE: {{ .Release.Name }} diff --git a/helm/plater/templates/memgraph-deployment.yaml b/helm/plater/templates/memgraph-deployment.yaml new file mode 100644 index 000000000..053a39e54 --- /dev/null +++ b/helm/plater/templates/memgraph-deployment.yaml @@ -0,0 +1,92 @@ +{{- if and (eq .Values.graph.backend "memgraph") (not .Values.graph.external) }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "plater.fullname" . }}-memgraph + labels: + {{- include "plater.labels" . | nindent 4 }} +spec: + serviceName: {{ include "plater.fullname" . }}-memgraph-service + selector: + matchLabels: + {{- include "plater.selectorLabels" . | nindent 6 }} + service-type: memgraph-database + template: + metadata: + labels: + {{- include "plater.selectorLabels" . | nindent 8 }} + service-type: memgraph-database + spec: + {{- if .Values.app.memgraph.dataUrl }} + initContainers: + - name: {{ include "plater.fullname" . }}-memgraph-download-container + image: curlimages/curl:8.16.0 + command: + - sh + - -c + - | + echo "Downloading Memgraph dump..." + curl -L -o /data/memgraph.cypher {{ .Values.app.memgraph.dataUrl }} + volumeMounts: + - name: {{ include "plater.fullname" . }}-memgraph-pvc + mountPath: /data + subPath: memgraph_data + resources: + {{- toYaml .Values.app.memgraph.initresources | nindent 12 }} + {{- end }} + + containers: + - name: {{ include "plater.fullname" . }}-memgraph-container + image: "{{ .Values.image.memgraph.repository }}:{{ .Values.image.memgraph.tag }}" + imagePullPolicy: {{ .Values.image.memgraph.imagePullPolicy }} + ports: + - name: memgraph-bolt + containerPort: {{ .Values.app.memgraph.boltPort }} + protocol: TCP + command: + - '/run_memgraph.sh' + {{- with .Values.app.memgraph.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: {{ include "plater.fullname" . }}-memgraph-pvc + mountPath: /data + subPath: memgraph_data + - name: {{ include "plater.fullname" . }}-scripts + mountPath: /run_memgraph.sh + subPath: run_memgraph.sh + startupProbe: +{{ toYaml .Values.app.memgraph.startupProbe | indent 12 }} + readinessProbe: +{{ toYaml .Values.app.memgraph.readinessProbe | indent 12 }} + livenessProbe: +{{ toYaml .Values.app.memgraph.livenessProbe | indent 12 }} + + restartPolicy: Always + {{ with .Values.app.memgraph.nodeSelector }} + nodeSelector: + {{ toYaml . | nindent 8 }} + {{ end }} + {{ with .Values.app.memgraph.affinity }} + affinity: + {{ toYaml . | nindent 8 }} + {{ end }} + {{ with .Values.app.memgraph.tolerations }} + tolerations: + {{ toYaml . | nindent 8 }} + {{ end }} + volumes: + - name: {{ include "plater.fullname" . }}-scripts + configMap: + name: {{ include "plater.fullname" . }}-configmap + defaultMode: 0777 + volumeClaimTemplates: + - metadata: + name: {{ include "plater.fullname" . }}-memgraph-pvc + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.app.memgraph.storage.size }} +{{ end }} diff --git a/helm/plater/templates/memgraph-service.yaml b/helm/plater/templates/memgraph-service.yaml new file mode 100644 index 000000000..0aa48e345 --- /dev/null +++ b/helm/plater/templates/memgraph-service.yaml @@ -0,0 +1,18 @@ +{{- if and (eq .Values.graph.backend "memgraph") (not .Values.graph.external) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "plater.fullname" . }}-memgraph-service + labels: + {{- include "plater.labels" . | nindent 4 }} +spec: + type: {{ .Values.app.memgraph.service.type }} + ports: + - port: {{ .Values.app.memgraph.boltPort }} + targetPort: {{ .Values.app.memgraph.boltPort }} + protocol: TCP + name: memgraph-bolt + selector: + {{- include "plater.selectorLabels" . | nindent 4 }} + service-type: memgraph-database +{{ end }} diff --git a/helm/plater/templates/neo4j-deployment.yaml b/helm/plater/templates/neo4j-deployment.yaml index 927874cfc..71eb52626 100644 --- a/helm/plater/templates/neo4j-deployment.yaml +++ b/helm/plater/templates/neo4j-deployment.yaml @@ -1,4 +1,4 @@ -{{ if not .Values.externalNeo4j }} +{{- if and (eq .Values.graph.backend "neo4j") (not .Values.graph.external) }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -10,19 +10,19 @@ spec: selector: matchLabels: {{- include "plater.selectorLabels" . | nindent 6 }} - service-type: database + service-type: neo4j-database template: metadata: labels: {{- include "plater.selectorLabels" . | nindent 8 }} - service-type: database + service-type: neo4j-database spec: {{- if .Values.dataUrl }} initContainers: - name: {{ include "plater.fullname" . }}-neo4j-download-container image: {{ .Values.image.neo4j.repository }}:{{ .Values.image.neo4j.tag }} command: - - '/download_data.sh' + - '/download_data_neo4j.sh' env: - name: NEO4J_DATA_URL value: {{ .Values.dataUrl }} @@ -33,8 +33,8 @@ spec: name: {{ include "plater.fullname" . }}-neo4jkp-pvc subPath: neo4j_data - name: {{ include "plater.fullname" . }}-scripts - mountPath: /download_data.sh - subPath: download_data.sh + mountPath: /download_data_neo4j.sh + subPath: download_data_neo4j.sh - name: {{ include "plater.fullname" . }}-neo4j-seed-container image: {{ .Values.image.neo4j.repository }}:{{ .Values.image.neo4j.tag }} command: @@ -88,7 +88,7 @@ spec: - name: NEO4J_server_jvm_additional value: "-XX:+ExitOnOutOfMemoryError -XX:+UseCompressedOops -Dlog4j2.disable.jmx=true -Dlog4j2.formatMsgNoLookups=true" - name: NEO4J_db_transaction_timeout - value: "{{ .Values.app.Neo4jQueryTimeout }}s" + value: "{{ .Values.app.GRAPH_QUERY_TIMEOUT }}s" ports: - name: neo4j-http containerPort: 7474 diff --git a/helm/plater/templates/neo4j-service.yaml b/helm/plater/templates/neo4j-service.yaml new file mode 100644 index 000000000..8ad899f7a --- /dev/null +++ b/helm/plater/templates/neo4j-service.yaml @@ -0,0 +1,22 @@ +{{- if and (eq .Values.graph.backend "neo4j") (not .Values.graph.external) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "plater.fullname" . }}-neo4j-service + labels: + {{- include "plater.labels" . | nindent 4 }} +spec: + type: {{ .Values.app.neo4j.service.type }} + ports: + - port: {{ .Values.app.neo4j.httpPort }} + targetPort: {{ .Values.app.neo4j.httpPort }} + protocol: TCP + name: neo4j-http + - port: {{ .Values.app.neo4j.boltPort }} + targetPort: {{ .Values.app.neo4j.boltPort }} + protocol: TCP + name: neo4j-bolt + selector: + {{- include "plater.selectorLabels" . | nindent 4 }} + service-type: neo4j-database +{{ end }} diff --git a/helm/plater/templates/service.yaml b/helm/plater/templates/service.yaml index 0db7c5d92..ebc48ce34 100644 --- a/helm/plater/templates/service.yaml +++ b/helm/plater/templates/service.yaml @@ -14,24 +14,3 @@ spec: selector: {{- include "plater.selectorLabels" . | nindent 4 }} service-type: web-server ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "plater.fullname" . }}-neo4j-service - labels: - {{- include "plater.labels" . | nindent 4 }} -spec: - type: {{ .Values.app.neo4j.service.type }} - ports: - - port: {{ .Values.app.neo4j.httpPort }} - targetPort: {{ .Values.app.neo4j.httpPort }} - protocol: TCP - name: neo4j-http - - port: {{ .Values.app.neo4j.boltPort }} - targetPort: {{ .Values.app.neo4j.boltPort }} - protocol: TCP - name: neo4j-bolt - selector: - {{- include "plater.selectorLabels" . | nindent 4 }} - service-type: database \ No newline at end of file diff --git a/helm/plater/values.yaml b/helm/plater/values.yaml index 0d0be5205..b9388a3ea 100644 --- a/helm/plater/values.yaml +++ b/helm/plater/values.yaml @@ -4,6 +4,9 @@ replicaCount: 1 +graph: + backend: memgraph # neo4j or memgraph + external: false busybox: image: @@ -18,12 +21,16 @@ bash: image: plater: repository: ghcr.io/translatorsri/plater-clustered - tag: "v2.1.4" + tag: "v2.3.0" imagePullPolicy: Always neo4j: repository: ghcr.io/translatorsri/plater-neo4j tag: "neo4j-5.26" imagePullPolicy: Always + memgraph: + repository: memgraph/memgraph + tag: "3.5.1" + pullPolicy: Always nameOverride: "" fullnameOverride: "" @@ -76,24 +83,24 @@ service: type: ClusterIP app: otel: - enabled: "True" + enabled: "False" jaegerHost: "http://jaeger-otel-collector" jaegerPort: "4317" - profiler_on: "True" + profiler_on: "False" gunicorn: worker_timeout: 3600 num_workers: 4 # this is the timeout used for the http request made to neo4j # and also controls the https://neo4j.com/docs/operations-manual/4.2/reference/configuration-settings/#config_dbms.transaction.timeout - Neo4jQueryTimeout: 1600 + GRAPH_QUERY_TIMEOUT: 1600 web: resources: requests: memory: 1Gi cpu: 500m limits: - memory: 3Gi - cpu: 2000m + memory: 2Gi + cpu: 1000m port: 8080 automatAddress: http://automat @@ -107,8 +114,8 @@ app: # additional jvm parameters for large query strings jvmStackSize: 1g # These are jvm based values hence suffix with one of the following G, M etc... - heapSize: - pageCacheSize: + heapSize: 1g + pageCacheSize: 1g # It's used to request the cluster for memory capable of running neo4j per the heap and PageCache values above # Compute this value by summing up heapSize + pageCacheSize + (1G for os) + jvmStackSize # https://neo4j.com/docs/operations-manual/current/performance/memory-configuration/ @@ -123,13 +130,15 @@ app: cpu: 500m limits: memory: 10Gi - cpu: 2000m + cpu: 1000m resources: requests: - cpu: 2000m + cpu: 1000m + memory: 5Gi limits: ephemeral-storage: 256Mi - cpu: 2500m + memory: 6Gi + cpu: 1500m openapi_config: # Adds more configs to default open api config x-trapi: @@ -157,12 +166,47 @@ app: # this value should match chart installation name # https://automat-dev.renci.org/ url: "" + + memgraph: + storage: + size: 20Mi + service: + type: ClusterIP + boltPort: 7687 + dataUrl: "" # URL to memgraph cypher file to populate the database, if deploying internally + initresources: + requests: + memory: 1Gi + cpu: 500m + limits: + memory: 1Gi + cpu: 1000m + resources: + requests: + memory: 5Gi + cpu: 500m + limits: + ephemeral-storage: 256Mi + memory: 6Gi + cpu: 1000m + nodeSelector: {} tolerations: [] affinity: {} +externalNeo4j: + hostName: "" + boltPort: 7687 + httpPort: 7474 + username: neo4j + password: "" + +externalMemgraph: + hostName: "" + boltPort: 7687 + skipAttributes: -dataUrl: "" +dataUrl: "" # this dataUrl is for neo4j data dump. Although it is better to move it to app.neo4j block, it could introduce breaking changes, so keep it here for now. metadataUrl: "" metaKGUrl: "" testingDataUrl: "" @@ -180,4 +224,4 @@ x_trapi: maturity: value: "maturity" location: - value: "location" \ No newline at end of file + value: "location"