WIP: introduce workflow pod

.gitignore

@@ -108,4 +108,35 @@ Composer-Setup.exe
 .metals/
 .bloop/
 .ammonite/
-metals.sbt
+metals.sbt
+
+
+# Ignore Helm-specific artifacts in texera-helmchart
+core/scripts/texera-helmchart/*.tgz
+core/scripts/texera-helmchart/charts/
+core/scripts/texera-helmchart/tmpcharts/
+core/scripts/texera-helmchart/*.lock
+core/scripts/texera-helmchart/test-output/
+core/scripts/texera-helmchart/tests/test-*.yaml
+core/scripts/texera-helmchart/*.helmignore
+
+# Ignore general OS-specific files and editor files in texera-helmchart
+core/scripts/texera-helmchart/.DS_Store
+core/scripts/texera-helmchart/Thumbs.db
+core/scripts/texera-helmchart/*.swp
+core/scripts/texera-helmchart/*.swo
+core/scripts/texera-helmchart/*.bak
+core/scripts/texera-helmchart/*~
+core/scripts/texera-helmchart/.idea/
+core/scripts/texera-helmchart/.vscode/
+core/scripts/texera-helmchart/*.log
+
+# Temporary files in texera-helmchart
+core/scripts/texera-helmchart/*.tmp
+core/scripts/texera-helmchart/*.out
+core/scripts/texera-helmchart/*.cache
+core/scripts/texera-helmchart/*.pyc
+core/scripts/texera-helmchart/__pycache__/
+
+# Build artifacts in texera-helmchart
+core/scripts/texera-helmchart/dist/

core/amber/computing-unit.dockerfile

@@ -0,0 +1,41 @@
+FROM sbtscala/scala-sbt:eclipse-temurin-jammy-11.0.17_8_1.9.3_2.13.11
+
+# Set working directory
+WORKDIR /core
+
+# Copy all projects under core to /core
+COPY core/ .
+
+RUN rm -rf amber/user-resources/*
+
+# Update system and install dependencies
+RUN apt-get update && apt-get install -y \
+    netcat \
+    unzip \
+    python3-pip \
+    libpq-dev \
+    python3-dev \
+    && apt-get clean
+
+# Install Python dependencies
+RUN pip3 install --upgrade pip setuptools wheel
+RUN pip3 install python-lsp-server python-lsp-server[websockets]
+
+# Install requirements with a fallback for wordcloud
+RUN pip3 install -r amber/requirements.txt
+RUN pip3 install --no-cache-dir --find-links https://pypi.org/simple/ -r amber/operator-requirements.txt || \
+    pip3 install --no-cache-dir wordcloud==1.9.2
+
+# Additional setup
+WORKDIR /core
+# Add .git for runtime calls to jgit from OPversion
+COPY .git ../.git
+
+# Build services
+RUN scripts/build-services.sh
+
+# Set the default command
+CMD ["scripts/workflow-computing-unit.sh"]
+
+# Expose the required port
+EXPOSE 8085

core/amber/src/main/scala/edu/uci/ics/amber/engine/common/AmberRuntime.scala

@@ -54,6 +54,7 @@ object AmberRuntime {
         akka.cluster.seed-nodes = [ "akka://Amber@$localIpAddress:2552" ]
         """)
       .withFallback(akkaConfig)
+      .resolve()
     AmberConfig.masterNodeAddr = createMasterAddress(localIpAddress)
     createAmberSystem(masterConfig)
   }
@@ -75,6 +76,7 @@ object AmberRuntime {
         akka.cluster.seed-nodes = [ "akka://Amber@$addr:2552" ]
         """)
       .withFallback(akkaConfig)
+      .resolve()
     AmberConfig.masterNodeAddr = createMasterAddress(addr)
     createAmberSystem(workerConfig)
   }

core/amber/src/main/scala/edu/uci/ics/texera/web/resource/ResultResource.scala

@@ -5,7 +5,7 @@ import edu.uci.ics.amber.core.virtualidentity.WorkflowIdentity
 import edu.uci.ics.texera.web.auth.SessionUser
 import edu.uci.ics.texera.web.model.websocket.request.ResultExportRequest
 import edu.uci.ics.texera.web.model.websocket.response.ResultExportResponse
-import edu.uci.ics.texera.web.service.{ResultExportService, WorkflowService}
+import edu.uci.ics.texera.web.service.ResultExportService
 import io.dropwizard.auth.Auth
 
 import javax.ws.rs._

core/amber/src/main/scala/edu/uci/ics/texera/web/service/ExecutionResultService.scala

@@ -4,7 +4,7 @@ import akka.actor.Cancellable
 import com.fasterxml.jackson.annotation.{JsonTypeInfo, JsonTypeName}
 import com.fasterxml.jackson.databind.node.ObjectNode
 import com.typesafe.scalalogging.LazyLogging
-import edu.uci.ics.amber.core.storage.DocumentFactory.{ICEBERG, MONGODB}
+import edu.uci.ics.amber.core.storage.DocumentFactory.ICEBERG
 import edu.uci.ics.amber.core.storage.VFSResourceType.MATERIALIZED_RESULT
 import edu.uci.ics.amber.core.storage.model.VirtualDocument
 import edu.uci.ics.amber.core.storage.{DocumentFactory, StorageConfig, VFSURIFactory}

core/amber/webserver.dockerfile

@@ -0,0 +1,42 @@
+FROM node:18-alpine AS nodegui
+
+WORKDIR /gui
+COPY core/gui/package.json core/gui/yarn.lock ./
+RUN apk add --no-cache python3 make g++ git
+RUN corepack enable && corepack prepare yarn@4.5.1 --activate && yarn set version --yarn-path  4.5.1
+# Fake git-version.js during yarn install to prevent git from causing cache
+# invalidation of dependencies
+# Add tools required for building dependencies
+RUN touch git-version.js && YARN_NODE_LINKER=node-modules yarn install
+
+COPY core/gui .
+# Position of .git doesn't matter since it's only there for the revision hash
+COPY .git ./.git
+RUN apk add --no-cache git && \
+	node git-version.js && \
+	apk del git && \
+	yarn run build
+
+FROM sbtscala/scala-sbt:eclipse-temurin-jammy-11.0.17_8_1.9.3_2.13.11
+
+# copy all projects under core to /core
+WORKDIR /core
+COPY core/ .
+
+RUN rm -rf amber/user-resources/*
+
+RUN apt-get update
+RUN apt-get install -y netcat unzip python3-pip libpq-dev python3-dev
+RUN pip3 install python-lsp-server python-lsp-server[websockets]
+RUN pip3 install -r amber/requirements.txt
+
+WORKDIR /core
+# Add .git for runtime calls to jgit from OPversion
+COPY .git ../.git
+COPY --from=nodegui /gui/dist ./gui/dist
+
+RUN scripts/build-services.sh
+
+CMD ["scripts/server.sh"]
+
+EXPOSE 8080

core/build.sbt

@@ -3,6 +3,14 @@ lazy val WorkflowCore = (project in file("workflow-core"))
   .dependsOn(DAO)
   .configs(Test)
   .dependsOn(DAO % "test->test") // test scope dependency
+lazy val WorkflowComputingUnitManagingService = (project in file("workflow-computing-unit-managing-service"))
+  .dependsOn(WorkflowCore)
+  .settings(
+    dependencyOverrides ++= Seq(
+      // override it as io.dropwizard 4 require 2.16.1 or higher
+      "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.17.0",
+    )
+  )
 lazy val WorkflowOperator = (project in file("workflow-operator"))
   .dependsOn(WorkflowCore)
   .settings(
@@ -39,7 +47,7 @@ lazy val WorkflowExecutionService = (project in file("amber"))
 
 // root project definition
 lazy val CoreProject = (project in file("."))
-  .aggregate(DAO, WorkflowCore, WorkflowOperator, WorkflowCompilingService, WorkflowExecutionService)
+  .aggregate(DAO, WorkflowComputingUnitManagingService, WorkflowCore, WorkflowOperator, WorkflowCompilingService, WorkflowExecutionService)
   .settings(
     name := "core",
     version := "0.1.0",

core/dao/build.sbt

@@ -89,4 +89,5 @@ libraryDependencies ++= Seq(
 libraryDependencies ++= Seq(
   "mysql" % "mysql-connector-java" % "8.0.33",          // MySQL connector
   "org.yaml" % "snakeyaml" % "1.30",                    // for reading storage config yaml file
+  "com.typesafe" % "config" % "1.4.3",                  // for reading storage config .conf file
 )

core/dao/src/main/scala/edu/uci/ics/texera/dao/JooqCodeGenerator.scala

@@ -2,12 +2,8 @@ package edu.uci.ics.texera.dao
 
 import org.jooq.codegen.GenerationTool
 import org.jooq.meta.jaxb.{Configuration, Jdbc}
-import org.yaml.snakeyaml.Yaml
-
-import java.io.InputStream
+import com.typesafe.config.{Config, ConfigFactory}
 import java.nio.file.{Files, Path}
-import java.util.{Map => JMap}
-import scala.jdk.CollectionConverters._
 
 object JooqCodeGenerator {
   @throws[Exception]
@@ -17,38 +13,33 @@ object JooqCodeGenerator {
       Path.of("dao").resolve("src").resolve("main").resolve("resources").resolve("jooq-conf.xml")
     val jooqConfig: Configuration = GenerationTool.load(Files.newInputStream(jooqXmlPath))
 
-    // Load YAML configuration
-    val yamlConfPath: Path = Path
+    // Load .conf configuration
+    val confPath: Path = Path
       .of("workflow-core")
       .resolve("src")
       .resolve("main")
       .resolve("resources")
-      .resolve("storage-config.yaml")
-    val yaml = new Yaml
-    val inputStream: InputStream = Files.newInputStream(yamlConfPath)
+      .resolve("storage.conf")
+    val conf: Config = ConfigFactory.parseFile(confPath.toFile)
 
-    val conf: Map[String, Any] =
-      yaml.load(inputStream).asInstanceOf[JMap[String, Any]].asScala.toMap
+    // Resolve placeholders in the configuration file
+    val resolvedConf: Config = conf.resolve()
 
-    val jdbcConfig = conf("storage")
-      .asInstanceOf[JMap[String, Any]]
-      .asScala("jdbc")
-      .asInstanceOf[JMap[String, Any]]
-      .asScala
+    // Get JDBC configuration from .conf file
+    val jdbcUsername = resolvedConf.getString("storage.jdbc.username")
+    val jdbcPassword = resolvedConf.getString("storage.jdbc.password")
+    val jdbcUrl = resolvedConf.getString("storage.jdbc.url")
 
     // Set JDBC configuration for jOOQ
     val jooqJdbcConfig = new Jdbc
     jooqJdbcConfig.setDriver("com.mysql.cj.jdbc.Driver")
-    jooqJdbcConfig.setUrl(jdbcConfig("url").toString)
-    jooqJdbcConfig.setUsername(jdbcConfig("username").toString)
-    jooqJdbcConfig.setPassword(jdbcConfig("password").toString)
+    jooqJdbcConfig.setUrl(jdbcUrl)
+    jooqJdbcConfig.setUsername(jdbcUsername)
+    jooqJdbcConfig.setPassword(jdbcPassword)
 
     jooqConfig.setJdbc(jooqJdbcConfig)
 
     // Generate the code
     GenerationTool.generate(jooqConfig)
-
-    // Close input stream
-    inputStream.close()
   }
 }