Containerised distribution of Guardian main branch

auth/app/auth/AuthComponents.scala

@@ -1,6 +1,6 @@
 package auth
 
-import com.gu.mediaservice.lib.management.{InnerServiceStatusCheckController, Management}
+import com.gu.mediaservice.lib.management.Management
 import com.gu.mediaservice.lib.play.GridComponents
 import play.api.ApplicationLoader.Context
 import play.api.{Configuration, Environment}
@@ -14,10 +14,9 @@ class AuthComponents(context: Context) extends GridComponents(context, new AuthC
 
   val controller = new AuthController(auth, providers, config, controllerComponents, authorisation)
   val permissionsAwareManagement = new Management(controllerComponents, buildInfo)
-  val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient)
 
 
-  override val router = new Routes(httpErrorHandler, controller, permissionsAwareManagement, InnerServiceStatusCheckController)
+  override val router = new Routes(httpErrorHandler, controller, permissionsAwareManagement)
 }
 
 object AuthHttpConfig {

auth/conf/routes

@@ -16,7 +16,6 @@ GET     /cookieMonster              auth.AuthController.cookieMonster
 # Management
 GET     /management/healthcheck                       com.gu.mediaservice.lib.management.Management.healthCheck
 GET     /management/manifest                          com.gu.mediaservice.lib.management.Management.manifest
-GET     /management/whoAmI                            com.gu.mediaservice.lib.management.InnerServiceStatusCheckController.whoAmI(depth: Int)
 
 # Shoo robots away
 GET     /robots.txt                                   com.gu.mediaservice.lib.management.Management.disallowRobots

build.sbt

@@ -4,8 +4,7 @@ import sbt.Package.FixedTimestamp
 import scala.sys.process._
 import scala.util.control.NonFatal
 import scala.collection.JavaConverters._
-
-import com.typesafe.sbt.packager.debian.JDebPackaging
+import com.typesafe.sbt.packager.docker._
 
 // We need to keep the timestamps to allow caching headers to work as expected on assets.
 // The below should work, but some problem in one of the plugins (possible the play plugin? or sbt-web?) causes
@@ -125,9 +124,9 @@ lazy val auth = playProject("auth", 9011)
 
 lazy val collections = playProject("collections", 9010)
 
-lazy val cropper = playProject("cropper", 9006)
+lazy val cropper = playImageLoaderProject("cropper", 9006)
 
-lazy val imageLoader = playProject("image-loader", 9003).settings {
+lazy val imageLoader = playImageLoaderProject("image-loader", 9003).settings {
   libraryDependencies ++= Seq(
     "org.apache.tika" % "tika-core" % "1.28.5",
     "com.drewnoakes" % "metadata-extractor" % "2.19.0"
@@ -233,39 +232,58 @@ val buildInfo = Seq(
 )
 
 def playProject(projectName: String, port: Int, path: Option[String] = None): Project = {
-  val commonProject = project(projectName, path)
-    .enablePlugins(PlayScala, JDebPackaging, SystemdPlugin, BuildInfoPlugin)
+  project(projectName, path)
+    .enablePlugins(PlayScala, BuildInfoPlugin, DockerPlugin)
     .dependsOn(restLib)
     .settings(commonSettings ++ buildInfo ++ Seq(
+      dockerBaseImage := "eclipse-temurin:11",
+      dockerExposedPorts in Docker := Seq(port),
       playDefaultPort := port,
-      debianPackageDependencies := Seq("java11-runtime-headless"),
-      Linux / maintainer := "Guardian Developers <dig.dev.software@theguardian.com>",
-      Linux / packageSummary := description.value,
-      packageDescription := description.value,
 
       bashScriptEnvConfigLocation := Some("/etc/environment"),
-      Debian / makeEtcDefault := None,
-      Debian / packageBin := {
-        val originalFileName = (Debian / packageBin).value
-        val (base, ext) = originalFileName.baseAndExt
-        val newBase = base.replace(s"_${version.value}_all","")
-        val newFileName = file(originalFileName.getParent) / s"$newBase.$ext"
-        IO.move(originalFileName, newFileName)
-        println(s"Renamed $originalFileName to $newFileName")
-        newFileName
-      },
       Universal / mappings ++= Seq(
         file("common-lib/src/main/resources/application.conf") -> "conf/application.conf",
         file("common-lib/src/main/resources/logback.xml") -> "conf/logback.xml"
       ),
       Universal / javaOptions ++= Seq(
         "-Dpidfile.path=/dev/null",
-        s"-Dconfig.file=/usr/share/$projectName/conf/application.conf",
-        s"-Dlogger.file=/usr/share/$projectName/conf/logback.xml",
-        "-J-Xlog:gc*",
-        s"-J-Xlog:gc:/var/log/$projectName/gc.log"
-      )
-    ))
-  //Add the BBC library dependency if defined
-  maybeBBCLib.fold(commonProject){commonProject.dependsOn(_)}
+        s"-Dconfig.file=/opt/docker/conf/application.conf",
+        s"-Dlogger.file=/opt/docker/conf/logback.xml",
+        "-XX:+PrintCommandLineFlags", "-XX:MaxRAMPercentage=50"
+      ))
+    )
+}
+
+def playImageLoaderProject(projectName: String, port: Int, path: Option[String] = None): Project = {
+  project(projectName, path)
+    .enablePlugins(PlayScala, BuildInfoPlugin, DockerPlugin)
+    .dependsOn(restLib)
+    .settings(commonSettings ++ buildInfo ++ Seq(
+      dockerBaseImage := "eclipse-temurin:11",
+      dockerExposedPorts in Docker := Seq(port),
+      dockerCommands ++= Seq(
+        Cmd("USER", "root"), Cmd("RUN", "apt-get", "update"),
+        Cmd("RUN", "apt-get", "install", "-y", "apt-utils"),
+        Cmd("RUN", "apt-get", "install", "-y", "graphicsmagick"),
+        Cmd("RUN", "apt-get", "install", "-y", "graphicsmagick-imagemagick-compat"),
+        Cmd("RUN", "apt-get", "install", "-y", "pngquant"),
+        Cmd("RUN", "apt-get", "install", "-y", "libimage-exiftool-perl")
+      ),
+      playDefaultPort := port,
+
+      bashScriptEnvConfigLocation := Some("/etc/environment"),
+      Universal / mappings ++= Seq(
+        file("common-lib/src/main/resources/application.conf") -> "conf/application.conf",
+        file("common-lib/src/main/resources/logback.xml") -> "conf/logback.xml",
+        file("image-loader/cmyk.icc") -> "cmyk.icc",
+        file("image-loader/facebook-TINYsRGB_c2.icc") -> "facebook-TINYsRGB_c2.icc",
+        file("image-loader/grayscale.icc") -> "grayscale.icc",
+        file("image-loader/srgb.icc") -> "srgb.icc"
+      ),
+      Universal / javaOptions ++= Seq(
+        "-Dpidfile.path=/dev/null",
+        s"-Dconfig.file=/opt/docker/conf/application.conf",
+        s"-Dlogger.file=/opt/docker/conf/logback.xml",
+        "-XX:+PrintCommandLineFlags"
+  )))
 }

cloudbuild.yaml

@@ -0,0 +1,68 @@
+options:
+  machineType: 'N1_HIGHCPU_8'
+steps:
+  - name: 'node:24-alpine'
+    entrypoint: 'npm'
+    dir: 'kahuna'
+    args: [ 'install' ]
+  - name: 'node:24-alpine'
+    entrypoint: 'npm'
+    dir: 'kahuna'
+    args: [ 'run', 'test' ]
+  - name: 'node:24-alpine'
+    entrypoint: 'npm'
+    dir: 'kahuna'
+    args: [ 'run', 'dist' ]
+
+  - name: 'gcr.io/$PROJECT_ID/scala-sbt:1.6.2-jdk-11'
+    args: ['docker:publishLocal']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['tag', 'auth:0.1', 'eu.gcr.io/$PROJECT_ID/auth:$BRANCH_NAME']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'eu.gcr.io/$PROJECT_ID/auth:$BRANCH_NAME']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['tag', 'cropper:0.1', 'eu.gcr.io/$PROJECT_ID/cropper:$BRANCH_NAME']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'eu.gcr.io/$PROJECT_ID/cropper:$BRANCH_NAME']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['tag', 'collections:0.1', 'eu.gcr.io/$PROJECT_ID/collections:$BRANCH_NAME']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'eu.gcr.io/$PROJECT_ID/collections:$BRANCH_NAME']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['tag', 'image-loader:0.1', 'eu.gcr.io/$PROJECT_ID/image-loader:$BRANCH_NAME']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'eu.gcr.io/$PROJECT_ID/image-loader:$BRANCH_NAME']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['tag', 'kahuna:0.1', 'eu.gcr.io/$PROJECT_ID/kahuna:$BRANCH_NAME']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'eu.gcr.io/$PROJECT_ID/kahuna:$BRANCH_NAME']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['tag', 'leases:0.1', 'eu.gcr.io/$PROJECT_ID/leases:$BRANCH_NAME']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'eu.gcr.io/$PROJECT_ID/leases:$BRANCH_NAME']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['tag', 'media-api:0.1', 'eu.gcr.io/$PROJECT_ID/media-api:$BRANCH_NAME']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'eu.gcr.io/$PROJECT_ID/media-api:$BRANCH_NAME']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['tag', 'metadata-editor:0.1', 'eu.gcr.io/$PROJECT_ID/metadata-editor:$BRANCH_NAME']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'eu.gcr.io/$PROJECT_ID/metadata-editor:$BRANCH_NAME']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['tag', 'thrall:0.1', 'eu.gcr.io/$PROJECT_ID/thrall:$BRANCH_NAME']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'eu.gcr.io/$PROJECT_ID/thrall:$BRANCH_NAME']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['tag', 'usage:0.1', 'eu.gcr.io/$PROJECT_ID/usage:$BRANCH_NAME']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'eu.gcr.io/$PROJECT_ID/usage:$BRANCH_NAME']

collections/app/CollectionsComponents.scala

@@ -1,4 +1,3 @@
-import com.gu.mediaservice.lib.management.InnerServiceStatusCheckController
 import com.gu.mediaservice.lib.play.GridComponents
 import controllers.{CollectionsController, ImageCollectionsController}
 import lib.{CollectionsConfig, CollectionsMetrics, Notifications}
@@ -15,8 +14,7 @@ class CollectionsComponents(context: Context) extends GridComponents(context, ne
 
   val collections = new CollectionsController(auth, config, store, controllerComponents)
   val imageCollections = new ImageCollectionsController(auth, config, notifications, controllerComponents)
-  val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient)
 
 
-  override val router = new Routes(httpErrorHandler, collections, imageCollections, management, InnerServiceStatusCheckController)
+  override val router = new Routes(httpErrorHandler, collections, imageCollections, management)
 }

collections/conf/routes

@@ -16,7 +16,6 @@ POST    /corrected-collections                          controllers.CollectionsC
 # Management
 GET     /management/healthcheck                         com.gu.mediaservice.lib.management.Management.healthCheck
 GET     /management/manifest                            com.gu.mediaservice.lib.management.Management.manifest
-GET     /management/whoAmI                              com.gu.mediaservice.lib.management.InnerServiceStatusCheckController.whoAmI(depth: Int)
 
 # Shoo robots away
 GET     /robots.txt                                     com.gu.mediaservice.lib.management.Management.disallowRobots

common-lib/src/main/resources/logback.xml

@@ -2,30 +2,6 @@
 
     <conversionRule conversionWord="coloredLevel" converterClass="play.api.libs.logback.ColoredLevel" />
 
-    <if condition='property("STAGE").contains("DEV")'>
-        <then>
-            <property name="LOGS_LOCATION" value="${application.home}/logs" />
-        </then>
-        <else>
-            <property name="LOGS_LOCATION" value="logs" />
-        </else>
-    </if>
-
-    <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>${LOGS_LOCATION}/application.log</file>
-
-        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-            <fileNamePattern>${LOGS_LOCATION}/application.log.%d{yyyy-MM-dd}.%i.gz</fileNamePattern>
-            <maxFileSize>100MB</maxFileSize>
-            <maxHistory>7</maxHistory>
-            <totalSizeCap>500MB</totalSizeCap>
-        </rollingPolicy>
-
-        <encoder>
-            <pattern>%date - [%level] - from %logger in %thread markers=%marker %n%message%n%xException%n</pattern>
-        </encoder>
-    </appender>
-
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
       <encoder class="net.logstash.logback.encoder.LogstashEncoder" />
     </appender>
@@ -39,12 +15,7 @@
     <logger name="request" level="INFO" />
 
     <root level="INFO">
-      <appender-ref ref="LOGFILE"/>
-      <if condition='!property("STAGE").contains("DEV")'>
-        <then>
-          <appender-ref ref="ASYNCSTDOUT"/>
-        </then>
-      </if>
+        <appender-ref ref="ASYNCSTDOUT"/>
     </root>
 
 </configuration>

common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala

@@ -3,7 +3,7 @@ package com.gu.mediaservice
 import java.net.URL
 import com.gu.mediaservice.GridClient.{Error, Found, NotFound, Response}
 import com.gu.mediaservice.lib.config.Services
-import com.gu.mediaservice.model.{Collection, Crop, Edits, Image, ImageMetadata, ImageStatusRecord, SyndicationRights}
+import com.gu.mediaservice.model.{Collection, Crop, Edits, Image, ImageMetadata, ImageStatusRecord, SourceImage, SyndicationRights}
 import com.gu.mediaservice.model.leases.LeasesByMedia
 import com.gu.mediaservice.model.usage.Usage
 import com.typesafe.scalalogging.LazyLogging
@@ -104,12 +104,13 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo
    * process before returning data.
    * See also https://www.playframework.com/documentation/2.6.x/ScalaWS#Configuring-Timeouts
    */
-  def makeGetRequestAsync(url: URL, authFn: WSRequest => WSRequest, requestTimeout: Option[Duration] = None)
+  def makeGetRequestAsync(url: URL, authFn: WSRequest => WSRequest, requestTimeout: Option[Duration] = None,
+                          queryStringParameters: Option[Seq[(String, String)]] = None)
                          (implicit ec: ExecutionContext): Future[Response] = {
-    val request: WSRequest = wsClient.url(url.toString)
+    val request: WSRequest = wsClient.url(url.toString).withQueryStringParameters(queryStringParameters.getOrElse(Seq.empty): _*)
     val requestWithTimeout = requestTimeout.fold(request)(request.withRequestTimeout)
     val authorisedRequest = authFn(requestWithTimeout)
-    authorisedRequest.get().map { response => validateResponse(response, url)}
+    authorisedRequest.get().map { response => validateResponse(response, url) }
   }
 
   private def validateResponse(
@@ -234,6 +235,16 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo
     }
   }
 
+  def getSourceImage(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext): Future[SourceImage] = {
+    logger.info("attempt to get image")
+    val url = new URL(s"${services.apiBaseUri}/images/$mediaId")
+    makeGetRequestAsync(url, authFn, queryStringParameters = Some(Seq("include" -> "fileMetadata"))) map {
+      case Found(json, _) => json.as[SourceImage]
+      case nf@NotFound(_, _) => Error(nf.status, url, nf.underlying).logErrorAndThrowException()
+      case e@Error(_, _, _) => e.logErrorAndThrowException()
+    }
+  }
+
   def getMetadata(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext): Future[ImageMetadata] = {
     logger.info("attempt to get metadata")
     val url = new URL(s"${services.apiBaseUri}/images/$mediaId")

common-lib/src/main/scala/com/gu/mediaservice/lib/auth/ApiAccessor.scala

@@ -29,6 +29,9 @@ object ApiAccessor extends ArgoHelpers {
   def hasAccess(apiKey: ApiAccessor, request: RequestHeader, services: Services): Boolean = apiKey.tier match {
     case Internal => true
     case ReadOnly => request.method == "GET"
-    case Syndication => request.method == "GET" && request.host == services.apiHost && request.path.startsWith("/images")
+    case Syndication => {
+      val isMediaApiRequest = request.uri.startsWith(services.apiBaseUri) // TODO check this!
+      request.method == "GET" && isMediaApiRequest && request.path.startsWith("/images")
+    }
   }
 }

common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV1BuilderUtils.scala

@@ -1,7 +1,7 @@
 package com.gu.mediaservice.lib.aws
 
 import com.amazonaws.auth.profile.ProfileCredentialsProvider
-import com.amazonaws.auth.{AWSCredentialsProvider, AWSCredentialsProviderChain, InstanceProfileCredentialsProvider}
+import com.amazonaws.auth.{AWSCredentialsProvider, AWSCredentialsProviderChain, EnvironmentVariableCredentialsProvider, InstanceProfileCredentialsProvider}
 import com.amazonaws.client.builder.AwsClientBuilder
 import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
 import com.gu.mediaservice.lib.logging.GridLogging
@@ -13,8 +13,7 @@ trait AwsClientV1BuilderUtils extends GridLogging {
   def awsRegion: String = "eu-west-1"
 
   def awsCredentials: AWSCredentialsProvider = new AWSCredentialsProviderChain(
-    new ProfileCredentialsProvider("media-service"),
-    InstanceProfileCredentialsProvider.getInstance()
+    new EnvironmentVariableCredentialsProvider(),
   )
 
   final def awsEndpointConfiguration: Option[EndpointConfiguration] = awsLocalEndpoint match {

common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala

@@ -60,24 +60,11 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
   val domainRoot: String = string("domain.root")
   val domainRootOverride: Option[String] = stringOpt("domain.root-override")
   val rootAppName: String = stringDefault("app.name.root", "media")
-  val serviceHosts = ServiceHosts(
-    stringDefault("hosts.kahunaPrefix", s"$rootAppName."),
-    stringDefault("hosts.apiPrefix", s"api.$rootAppName."),
-    stringDefault("hosts.loaderPrefix", s"loader.$rootAppName."),
-    stringDefault("hosts.projectionPrefix", s"loader-projection.$rootAppName."),
-    stringDefault("hosts.cropperPrefix", s"cropper.$rootAppName."),
-    stringDefault("hosts.metadataPrefix", s"$rootAppName-metadata."),
-    stringDefault("hosts.imgopsPrefix", s"$rootAppName-imgops."),
-    stringDefault("hosts.usagePrefix", s"$rootAppName-usage."),
-    stringDefault("hosts.collectionsPrefix", s"$rootAppName-collections."),
-    stringDefault("hosts.leasesPrefix", s"$rootAppName-leases."),
-    stringDefault("hosts.authPrefix", s"$rootAppName-auth."),
-    stringDefault("hosts.thrallPrefix", s"thrall.$rootAppName.")
-  )
 
   val corsAllowedOrigins: Set[String] = getStringSet("security.cors.allowedOrigins")
 
-  val services = new Services(domainRoot, serviceHosts, corsAllowedOrigins, domainRootOverride)
+  private val singleHostUrl: String = string("single.host.url")
+  val services = new SingleHostServices(singleHostUrl)
 
   /**
    * Load in a list of domain metadata specifications from configuration. For example:

common-lib/src/main/scala/com/gu/mediaservice/lib/config/GridConfigLoader.scala

@@ -57,7 +57,10 @@ object GridConfigLoader extends StrictLogging {
       if (file.getPath.endsWith(".properties")) {
         logger.warn(s"Configuring the Grid with Java properties files is deprecated as of #3011, please switch to .conf files. See #3037 for a conversion utility.")
       }
-      Configuration(ConfigFactory.parseFile(file))
+      val parsed = ConfigFactory.parseFile(file)
+      logger.info(s"Resolving config parsed from file: $file")
+      val resolved = parsed.resolve()
+      Configuration(resolved)
     } else {
       logger.info(s"Skipping config file $file as it doesn't exist")
       Configuration.empty