From fdf1a8d1fd9e698136917e5c2399b36a244c0213 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 15 Dec 2024 15:00:53 +0000
Subject: [PATCH 01/45] Push raw s3 client usages to the S3 trait. S3 client
 field now private

Protects usages from the specific of which client is used to access which bucket.
---
 .../com/gu/mediaservice/lib/BaseStore.scala   |  5 +-
 .../lib/ImageIngestOperations.scala           |  8 +--
 .../gu/mediaservice/lib/S3ImageStorage.scala  | 10 ++--
 .../gu/mediaservice/lib/auth/KeyStore.scala   |  5 +-
 .../com/gu/mediaservice/lib/aws/S3.scala      | 56 ++++++++++++++++++-
 image-loader/app/lib/ImageLoaderStore.scala   |  8 +--
 media-api/app/lib/S3Client.scala              |  2 +-
 thrall/app/controllers/ReaperController.scala | 14 +++--
 8 files changed, 80 insertions(+), 28 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala
index 0aeb698ccc..2628508c7e 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala
@@ -25,15 +25,14 @@ abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonCon
   protected def getS3Object(key: String): Option[String] = s3.getObjectAsString(bucket, key)
 
   protected def getLatestS3Stream: Option[InputStream] = {
-    val objects = s3.client
-      .listObjects(bucket).getObjectSummaries.asScala
+    val objects = s3.listObjects(bucket).getObjectSummaries.asScala
       .filterNot(_.getKey == "AMAZON_SES_SETUP_NOTIFICATION")
 
     if (objects.nonEmpty) {
       val obj = objects.maxBy(_.getLastModified)
       logger.info(s"Latest key ${obj.getKey} in bucket $bucket")
 
-      val stream = s3.client.getObject(bucket, obj.getKey).getObjectContent
+      val stream = s3.getObject(bucket, obj).getObjectContent
       Some(stream)
     } else {
       logger.error(s"Bucket $bucket is empty")
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index 96c9412556..43ba653625 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -1,6 +1,6 @@
 package com.gu.mediaservice.lib
 
-import com.amazonaws.services.s3.model.{DeleteObjectsRequest, MultiObjectDeleteException}
+import com.amazonaws.services.s3.model.MultiObjectDeleteException
 
 import java.io.File
 import com.gu.mediaservice.lib.config.CommonConfig
@@ -62,9 +62,7 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
     case _ => Future {
       try {
         logger.info(s"Creating S3 bulkDelete request for $bucket / keys: " + keys.mkString(","))
-        client.deleteObjects(
-          new DeleteObjectsRequest(bucket).withKeys(keys: _*)
-        )
+        deleteObjects(bucket, keys)
         keys.map { key =>
           key -> true
         }.toMap
@@ -87,7 +85,7 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
   def deletePNGs(ids: Set[String])(implicit instance: Instance) = bulkDelete(imageBucket, ids.map(id => optimisedPngKeyFromId(id)).toList)
 
   def doesOriginalExist(id: String)(implicit instance: Instance): Boolean =
-    client.doesObjectExist(imageBucket, fileKeyFromId(id))
+    doesObjectExist(imageBucket, fileKeyFromId(id))
 
   private def instanceAwareOriginalImageKey(storableImage: StorableOriginalImage) = {
     fileKeyFromId(storableImage.id)(storableImage.instance)
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
index 30b5d4b21c..1c7298a3d3 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
@@ -28,20 +28,20 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage
   }
 
   def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future {
-    client.deleteObject(bucket, id)
+    deleteObject(bucket, id)
     logger.info(logMarker, s"Deleted image $id from bucket $bucket")
   }
 
   def deleteVersionedImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future {
-    val objectVersion = client.getObjectMetadata(bucket, id).getVersionId
-    client.deleteVersion(bucket, id, objectVersion)
+    val objectVersion = getObjectMetadata(bucket, id).getVersionId
+    deleteVersion(bucket, id, objectVersion)
     logger.info(logMarker, s"Deleted image $id from bucket $bucket (version: $objectVersion)")
   }
 
   def deleteFolder(bucket: String, id: String)(implicit logMarker: LogMarker) = Future {
-		val files = client.listObjects(bucket, id).getObjectSummaries.asScala
+		val files = listObjects(bucket, id).getObjectSummaries.asScala
     logger.info(s"Found ${files.size} files to delete in folder $id")
-    files.foreach(file => client.deleteObject(bucket, file.getKey))
+    files.foreach(file => deleteObject(bucket, file.getKey))
 		logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket")
 	}
 
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala
index bbb9da0ca1..ba98bcd177 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala
@@ -4,7 +4,6 @@ import com.gu.mediaservice.lib.BaseStore
 import com.gu.mediaservice.lib.config.CommonConfig
 import com.gu.mediaservice.model.Instance
 
-import scala.jdk.CollectionConverters._
 import scala.concurrent.ExecutionContext
 
 class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionContext)
@@ -17,7 +16,7 @@ class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionConte
   }
 
   private def fetchAll: Map[String, ApiAccessor] = {
-    val keys = s3.client.listObjects(bucket).getObjectSummaries.asScala.map(_.getKey)
-    keys.flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap
+    s3.listObjectKeys(bucket).flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap
   }
+
 }
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index 04628b72e7..bdf2da0133 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -65,7 +65,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
   type Key = String
   type UserMetadata = Map[String, String]
 
-  lazy val client: AmazonS3 = S3Ops.buildS3Client(config)
+  private lazy val client: AmazonS3 = S3Ops.buildS3Client(config)
   // also create a legacy client that uses v2 signatures for URL signing
   private lazy val legacySigningClient: AmazonS3 = S3Ops.buildS3Client(config, forceV2Sigs = true)
 
@@ -89,12 +89,46 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     legacySigningClient.generatePresignedUrl(request)
   }
 
+  def copyObject(sourceBucket: Bucket, destinationBucket: Bucket, key: String): CopyObjectResult = {
+    client.copyObject(sourceBucket, key, destinationBucket, key)
+  }
+
+  def generatePresignedRequest(request: GeneratePresignedUrlRequest): URL = {
+    client.generatePresignedUrl(request)
+  }
+
+  def deleteObject(bucket: Bucket, key: String): Unit = {
+    client.deleteObject(bucket, key)
+  }
+
+  def deleteObjects(bucket: Bucket, keys: Seq[Bucket]): DeleteObjectsResult = {
+    client.deleteObjects(
+      new DeleteObjectsRequest(bucket).withKeys(keys: _*)
+    )
+  }
+
+  def deleteVersion(bucket: Bucket, id: Bucket, objectVersion: Bucket): Unit = {
+    client.deleteVersion(bucket, id, objectVersion)
+  }
+
+  def doesObjectExist(bucket: Bucket, key: String) = {
+    client.doesObjectExist(bucket, key)
+  }
+
   def getObject(bucket: Bucket, url: URI): model.S3Object = {
     // get path and remove leading `/`
     val key: Key = url.getPath.drop(1)
     client.getObject(new GetObjectRequest(bucket, key))
   }
 
+  def getObject(bucket: Bucket, key: String): model.S3Object = {
+    client.getObject(new GetObjectRequest(bucket, key))
+  }
+
+  def getObject(bucket: Bucket, obj: S3ObjectSummary): model.S3Object = {
+    client.getObject(bucket, obj.getKey)
+  }
+
   def getObjectAsString(bucket: Bucket, key: String): Option[String] = {
     val content = client.getObject(new GetObjectRequest(bucket, key))
     val stream = content.getObjectContent
@@ -110,6 +144,26 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     }
   }
 
+  def getObjectMetadata(bucket: Bucket, id: Bucket): ObjectMetadata = {
+    client.getObjectMetadata(bucket, id)
+  }
+
+  def listObjects(bucket: String): ObjectListing = {
+    client.listObjects(bucket)
+  }
+
+  def listObjects(bucket: String, prefix: String): ObjectListing = {
+    client.listObjects(bucket, prefix)
+  }
+
+  def listObjectKeys(bucket: String): Seq[String] = {
+    client.listObjects(bucket).getObjectSummaries.asScala.map(_.getKey).toSeq
+  }
+
+  def putObject(bucket: String, key: String, content: String): Unit = {
+    client.putObject(bucket, key, content)
+  }
+
   def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None)
            (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] =
     Future {
diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala
index 367254a822..4a04b04156 100644
--- a/image-loader/app/lib/ImageLoaderStore.scala
+++ b/image-loader/app/lib/ImageLoaderStore.scala
@@ -28,7 +28,7 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati
   }
 
   def getS3Object(key: String)(implicit logMarker: LogMarker): S3Object = handleNotFound(key) {
-    client.getObject(config.maybeIngestBucket.get, key)
+    getObject(config.maybeIngestBucket.get, key)
   } {
     logger.error(logMarker, s"Attempted to read $key from ingest bucket, but it does not exist.")
   }
@@ -44,18 +44,18 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati
     // sent by the client in manager.js
     request.putCustomRequestHeader("x-amz-meta-media-id", mediaId)
 
-    client.generatePresignedUrl(request).toString
+    generatePresignedRequest(request).toString
   }
 
   def moveObjectToFailedBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key){
-    client.copyObject(config.maybeIngestBucket.get, key, config.maybeFailBucket.get, key) // TODO Naked get - make optional
+    copyObject(config.maybeIngestBucket.get, config.maybeFailBucket.get, key) // TODO Naked get - make optional
     deleteObjectFromIngestBucket(key)
   } {
     logger.warn(logMarker, s"Attempted to copy $key from ingest bucket to fail bucket, but it does not exist.")
   }
 
   def deleteObjectFromIngestBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key) {
-    client.deleteObject(config.maybeIngestBucket.get,key)
+    deleteObject(config.maybeIngestBucket.get, key)
   } {
     logger.warn(logMarker, s"Attempted to delete $key from ingest bucket, but it does not exist.")
   }
diff --git a/media-api/app/lib/S3Client.scala b/media-api/app/lib/S3Client.scala
index 1161392cfd..a286d5420a 100644
--- a/media-api/app/lib/S3Client.scala
+++ b/media-api/app/lib/S3Client.scala
@@ -29,7 +29,7 @@ class S3Client(config: MediaApiConfig) extends S3(config) with CloudFrontDistrib
   lazy val keyPairId: Option[String] = config.cloudFrontKeyPairId
   lazy val privateKey: PrivateKey = {
     config.cloudFrontPrivateKeyBucket.flatMap(bucket => config.cloudFrontPrivateKeyBucketKey.map { key =>
-      val privateKeyStream = client.getObject(bucket, key).getObjectContent
+      val privateKeyStream = getObject(bucket, key).getObjectContent
       try {
         PEM.readPrivateKey(privateKeyStream)
       }
diff --git a/thrall/app/controllers/ReaperController.scala b/thrall/app/controllers/ReaperController.scala
index 711fabd547..51818771ba 100644
--- a/thrall/app/controllers/ReaperController.scala
+++ b/thrall/app/controllers/ReaperController.scala
@@ -114,7 +114,7 @@ class ReaperController(
         case Some(reaperBucket) =>
           val now = DateTime.now(DateTimeZone.UTC)
           val key = s"$deleteType/${s3DirNameFromDate(now)}/$deleteType-${now.toString()}.json"
-          store.client.putObject(reaperBucket, key, json.toString())
+          store.putObject(reaperBucket, key, json.toString())
           json
         }
       }
@@ -198,8 +198,8 @@ class ReaperController(
     case (Some(reaperBucket), Some(countOfImagesToReap)) =>
       val recentRecords = List(now, now.minusDays(1), now.minusDays(2)).flatMap { day =>
         val s3DirName = s3DirNameFromDate(day)
-        store.client.listObjects(reaperBucket, s"soft/$s3DirName/").getObjectSummaries.asScala.toList ++
-          store.client.listObjects(reaperBucket, s"hard/$s3DirName/").getObjectSummaries.asScala.toList
+        store.listObjects(reaperBucket, s"soft/$s3DirName/").getObjectSummaries.asScala.toList ++
+          store.listObjects(reaperBucket, s"hard/$s3DirName/").getObjectSummaries.asScala.toList
       }
 
       val recentRecordKeys = recentRecords
@@ -214,9 +214,11 @@ class ReaperController(
   def reaperRecord(key: String) = auth { config.maybeReaperBucket match {
     case None => NotImplemented("Reaper bucket not configured")
     case Some(reaperBucket) =>
-      Ok(
-        store.client.getObjectAsString(reaperBucket, key)
-      ).as(JSON)
+      store.getObjectAsString(reaperBucket, key).map { record =>
+        Ok(record).as(JSON)
+      }.getOrElse{
+        NotFound
+      }
   }}
 
   def conf() = Action.async {

From bd67668938af6c6e9ca0c12a3eaad7c07b182156 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Thu, 21 Nov 2024 21:35:11 +0000
Subject: [PATCH 02/45] Push hardcoded S3 endpoint up to config. Sets up for
 use of non AWS S3 buckets. Allow S3 service endpoint to be set from config
 overriding s3.awazonaws.com.

Allows buckets to be switched to a non AWS S3 provider in the future.
---
 .../lib/ImageIngestOperations.scala           | 15 +++++++-----
 .../com/gu/mediaservice/lib/aws/S3.scala      | 24 +++++++++----------
 .../lib/config/CommonConfig.scala             |  2 ++
 image-loader/app/model/Projector.scala        |  6 ++---
 image-loader/app/model/Uploader.scala         |  6 +++--
 .../test/scala/model/ImageUploadTest.scala    |  4 ++--
 .../test/scala/model/ProjectorTest.scala      |  2 +-
 7 files changed, 32 insertions(+), 27 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index 43ba653625..a4e72e7377 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -105,35 +105,38 @@ sealed trait ImageWrapper {
   val instance: Instance
 }
 sealed trait StorableImage extends ImageWrapper {
-  def toProjectedS3Object(thumbBucket: String): S3Object = S3Object(
+  def toProjectedS3Object(thumbBucket: String, s3Endpoint: String): S3Object = S3Object(
     thumbBucket,
     ImageIngestOperations.fileKeyFromId(id)(instance),
     file,
     Some(mimeType),
     lastModified = None,
-    meta
+    meta,
+    s3Endpoint = s3Endpoint
   )
 }
 
 case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage
 case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: DateTime, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage {
-  override def toProjectedS3Object(thumbBucket: String): S3Object = S3Object(
+  override def toProjectedS3Object(thumbBucket: String, s3Endpoint: String): S3Object = S3Object(
     thumbBucket,
     ImageIngestOperations.fileKeyFromId(id)(instance),
     file,
     Some(mimeType),
     lastModified = Some(lastModified),
-    meta
+    meta,
+    s3Endpoint = s3Endpoint
   )
 }
 case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage {
-  override def toProjectedS3Object(thumbBucket: String): S3Object = S3Object(
+  override def toProjectedS3Object(thumbBucket: String, s3Endpoint: String): S3Object = S3Object(
     thumbBucket,
     ImageIngestOperations.optimisedPngKeyFromId(id)(instance),
     file,
     Some(mimeType),
     lastModified = None,
-    meta = meta
+    meta = meta,
+    s3Endpoint = s3Endpoint
   )
 }
 
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index bdf2da0133..f4d1d02ce0 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -17,16 +17,16 @@ import scala.concurrent.{ExecutionContext, Future}
 case class S3Object(uri: URI, size: Long, metadata: S3Metadata)
 
 object S3Object {
-  def objectUrl(bucket: String, key: String): URI = {
-    val bucketUrl = s"$bucket.${S3Ops.s3Endpoint}"
+  private def objectUrl(bucket: String, key: String, s3Endpoint: String): URI = {
+    val bucketUrl = s"$bucket.$s3Endpoint"
     new URI("http", bucketUrl, s"/$key", null)
   }
 
-  def apply(bucket: String, key: String, size: Long, metadata: S3Metadata): S3Object =
-    apply(objectUrl(bucket, key), size, metadata)
+  def apply(bucket: String, key: String, size: Long, metadata: S3Metadata, s3Endpoint: String): S3Object =
+    apply(objectUrl(bucket, key, s3Endpoint), size, metadata)
 
   def apply(bucket: String, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime],
-            meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None): S3Object = {
+            meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None, s3Endpoint: String): S3Object = {
     S3Object(
       bucket,
       key,
@@ -38,7 +38,8 @@ object S3Object {
           cacheControl,
           lastModified
         )
-      )
+      ),
+      s3Endpoint
     )
   }
 }
@@ -65,6 +66,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
   type Key = String
   type UserMetadata = Map[String, String]
 
+  private val s3Endpoint: String = config.s3Endpoint
   private lazy val client: AmazonS3 = S3Ops.buildS3Client(config)
   // also create a legacy client that uses v2 signatures for URL signing
   private lazy val legacySigningClient: AmazonS3 = S3Ops.buildS3Client(config, forceV2Sigs = true)
@@ -184,7 +186,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
         client.putObject(req)
         // once we've completed the PUT read back to ensure that we are returning reality
         val metadata = client.getObjectMetadata(bucket, id)
-        S3Object(bucket, id, metadata.getContentLength, S3Metadata(metadata))
+        S3Object(bucket, id, metadata.getContentLength, S3Metadata(metadata), s3Endpoint)
       }(markers)
     }
 
@@ -198,7 +200,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     }.flatMap {
       case Some(objectMetadata) =>
         logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket $bucket")
-        Future.successful(S3Object(bucket, id, objectMetadata.getContentLength, S3Metadata(objectMetadata)))
+        Future.successful(S3Object(bucket, id, objectMetadata.getContentLength, S3Metadata(objectMetadata) ,s3Endpoint))
       case None =>
         store(bucket, id, file, mimeType, meta, cacheControl)
     }
@@ -212,7 +214,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
       val summaries = listing.getObjectSummaries.asScala
       summaries.map(summary => (summary.getKey, summary)).foldLeft(List[S3Object]()) {
         case (memo: List[S3Object], (key: String, summary: S3ObjectSummary)) =>
-          S3Object(bucket, key, summary.getSize, getMetadata(bucket, key)) :: memo
+          S3Object(bucket, key, summary.getSize, getMetadata(bucket, key), s3Endpoint) :: memo
       }
     }
 
@@ -234,10 +236,6 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
 }
 
 object S3Ops {
-  // TODO make this localstack friendly
-  // TODO: Make this region aware - i.e. RegionUtils.getRegion(region).getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX)
-  val s3Endpoint = "s3.amazonaws.com"
-
   def buildS3Client(config: CommonConfig, forceV2Sigs: Boolean = false, localstackAware: Boolean = true, maybeRegionOverride: Option[String] = None): AmazonS3 = {
 
     val clientConfig = new ClientConfiguration()
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
index 51adb4793f..4b1e3130de 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
@@ -58,6 +58,8 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
 
   val maybeUploadLimitInBytes: Option[Int] = intOpt("upload.limit.mb").map(_ * 1024 * 1024)
 
+  val s3Endpoint: String= stringOpt("s3.serviceEndpoint").getOrElse("s3.amazonaws.com")
+
   val instancesEndpoint: String = string("instance.service.instances")
 
   // Note: had to make these lazy to avoid init order problems ;_;
diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala
index 2be26928a3..e89c6a19ee 100644
--- a/image-loader/app/model/Projector.scala
+++ b/image-loader/app/model/Projector.scala
@@ -184,13 +184,13 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
   }
 
   private def projectOriginalFileAsS3Model(storableOriginalImage: StorableOriginalImage) =
-    Future.successful(storableOriginalImage.toProjectedS3Object(config.originalFileBucket))
+    Future.successful(storableOriginalImage.toProjectedS3Object(config.originalFileBucket, config.s3Endpoint))
 
   private def projectThumbnailFileAsS3Model(storableThumbImage: StorableThumbImage) =
-    Future.successful(storableThumbImage.toProjectedS3Object(config.thumbBucket))
+    Future.successful(storableThumbImage.toProjectedS3Object(config.thumbBucket, config.s3Endpoint))
 
   private def projectOptimisedPNGFileAsS3Model(storableOptimisedImage: StorableOptimisedImage) =
-    Future.successful(storableOptimisedImage.toProjectedS3Object(config.originalFileBucket))
+    Future.successful(storableOptimisedImage.toProjectedS3Object(config.originalFileBucket, config.s3Endpoint))
 
   private def fetchThumbFile(
     imageId: String, outFile: File, instance: Instance)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala
index c10cfa6013..fcfda68a9e 100644
--- a/image-loader/app/model/Uploader.scala
+++ b/image-loader/app/model/Uploader.scala
@@ -71,7 +71,8 @@ case class ImageUploadOpsCfg(
   thumbQuality: Double,
   transcodedMimeTypes: List[MimeType],
   originalFileBucket: String,
-  thumbBucket: String
+  thumbBucket: String,
+  s3Endpoint: String
 )
 
 case class ImageUploadOpsDependencies(
@@ -97,7 +98,8 @@ object Uploader extends GridLogging {
       config.thumbQuality,
       config.transcodedMimeTypes,
       config.imageBucket,
-      config.thumbnailBucket
+      config.thumbnailBucket,
+      config.s3Endpoint
     )
   }
 
diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala
index 0247f56d15..9721919ffb 100644
--- a/image-loader/test/scala/model/ImageUploadTest.scala
+++ b/image-loader/test/scala/model/ImageUploadTest.scala
@@ -32,7 +32,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
   private implicit val logMarker: MockLogMarker = new MockLogMarker()
     // For mime type info, see https://github.com/guardian/grid/pull/2568
     val tempDir = new File("/tmp")
-    val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), "img-bucket", "thumb-bucket")
+    val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), "img-bucket", "thumb-bucket", "s3.amazonaws.com")
 
   /**
     * @todo: I flailed about until I found a path that worked, but
@@ -53,7 +53,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
 
     def mockStore = (a: StorableImage) =>
       Future.successful(
-        S3Object("madeupname", "madeupkey", a.file, Some(a.mimeType), None, a.meta, None)
+        S3Object("madeupname", "madeupkey", a.file, Some(a.mimeType), None, a.meta, None, "s3.amazonaws.com")
       )
 
     def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore
diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala
index e7e5365d09..1343d92802 100644
--- a/image-loader/test/scala/model/ProjectorTest.scala
+++ b/image-loader/test/scala/model/ProjectorTest.scala
@@ -38,7 +38,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc
 
   private val imageOperations = new ImageOperations(ctxPath)
 
-  private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", "thumb-bucket")
+  private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", "thumb-bucket", "s3.amazonaws.com")
 
   private val s3 = mock[AmazonS3]
   private val auth = mock[Authentication]

From babc50298fd9e3b1c80c9121a418f9f901a87e6b Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 29 Nov 2024 16:04:26 +0000
Subject: [PATCH 03/45] Push s3 end point to all s3 users. Bucket specific end
 points.

Pull image and thumbnail buckets to common config.
---
 .../scala/com/gu/mediaservice/lib/BaseStore.scala   |  4 +++-
 .../gu/mediaservice/lib/ImageIngestOperations.scala |  8 ++++----
 .../lib/ImageQuarantineOperations.scala             |  4 ++--
 .../com/gu/mediaservice/lib/ImageStorage.scala      |  2 +-
 .../com/gu/mediaservice/lib/S3ImageStorage.scala    |  6 +++---
 .../com/gu/mediaservice/lib/auth/KeyStore.scala     |  4 ++--
 .../main/scala/com/gu/mediaservice/lib/aws/S3.scala | 13 ++++++-------
 .../gu/mediaservice/lib/config/CommonConfig.scala   |  7 +++++--
 common-lib/src/test/resources/application.conf      |  4 ++++
 cropper/app/lib/CropStore.scala                     |  4 ++--
 cropper/app/lib/CropperConfig.scala                 |  3 +--
 image-loader/app/lib/ImageLoaderConfig.scala        |  4 ++--
 image-loader/app/lib/ImageLoaderStore.scala         |  2 +-
 image-loader/app/lib/QuarantineStore.scala          |  2 +-
 image-loader/app/model/Uploader.scala               |  2 +-
 media-api/app/lib/ImageResponse.scala               |  2 +-
 media-api/app/lib/MediaApiConfig.scala              |  8 +++-----
 media-api/app/lib/QuotaStore.scala                  |  5 +++--
 media-api/app/lib/UsageQuota.scala                  |  6 ++++--
 media-api/app/lib/UsageStore.scala                  |  5 +++--
 .../provider/ApiKeyAuthenticationProvider.scala     |  2 +-
 .../lib/auth/ApiKeyAuthenticationProviderTest.scala |  2 +-
 .../gu/mediaservice/lib/JsonValueCodecJsValue.scala |  1 +
 thrall/app/lib/ThrallConfig.scala                   |  4 ----
 thrall/app/lib/ThrallStore.scala                    |  2 +-
 25 files changed, 56 insertions(+), 50 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala
index 2628508c7e..31d3cbb610 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala
@@ -14,9 +14,11 @@ import scala.concurrent.duration._
 import scala.util.control.NonFatal
 
 
-abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonConfig)(implicit ec: ExecutionContext)
+abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonConfig, s3Endpoint: String)(implicit ec: ExecutionContext)
   extends GridLogging {
 
+  def s3Endpoint: String
+
   val s3 = new S3(config)
 
   protected val store: AtomicReference[Map[TStoreKey, TStoreVal]] = new AtomicReference(Map.empty)
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index a4e72e7377..fb8e797204 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -21,7 +21,7 @@ object ImageIngestOperations {
   private def snippetForId(id: String) = id.take(6).mkString("/") + "/" + id
 }
 
-class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config: CommonConfig, isVersionedS3: Boolean = false)
+class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config: CommonConfig, isVersionedS3: Boolean = false, imageBucketS3Endpoint: String, thumbnailBucketS3Endpoint: String)
   extends S3ImageStorage(config) with StrictLogging {
 
   import ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId}
@@ -38,7 +38,7 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
     val instanceSpecificKey = instanceAwareOriginalImageKey(storableImage)
     logger.info(s"Storing original image to instance specific key:$imageBucket / $instanceSpecificKey")
     storeImage(imageBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType),
-      storableImage.meta, overwrite = false)
+      storableImage.meta, overwrite = false, s3Endpoint = imageBucketS3Endpoint)
   }
 
   private def storeThumbnailImage(storableImage: StorableThumbImage)
@@ -46,7 +46,7 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
     val instanceSpecificKey = instanceAwareThumbnailImageKey(storableImage)
     logger.info(s"Storing thumbnail to instance specific key: $thumbnailBucket / $instanceSpecificKey")
     storeImage(thumbnailBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType),
-      overwrite = true)
+      overwrite = true, s3Endpoint = thumbnailBucketS3Endpoint)
   }
 
   private def storeOptimisedImage(storableImage: StorableOptimisedImage)
@@ -54,7 +54,7 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
     val instanceSpecificKey = optimisedPngKeyFromId(storableImage.id)(storableImage.instance)
     logger.info(s"Storing optimised image to instance specific key: $thumbnailBucket / $instanceSpecificKey")
     storeImage(imageBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType),
-      overwrite = true)
+      overwrite = true, s3Endpoint = imageBucketS3Endpoint)
   }
 
   private def bulkDelete(bucket: String, keys: List[String]): Future[Map[String, Boolean]] = keys match {
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala
index 0cc3a146a0..51b04e980d 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala
@@ -8,12 +8,12 @@ import com.gu.mediaservice.model.{Instance, MimeType}
 
 import scala.concurrent.Future
 
-class ImageQuarantineOperations(quarantineBucket: String, config: CommonConfig, isVersionedS3: Boolean = false)
+class ImageQuarantineOperations(quarantineBucket: String, config: CommonConfig, isVersionedS3: Boolean = false, s3Endpoint: String)
   extends S3ImageStorage(config) {
 
   def storeQuarantineImage(id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty)
                        (implicit logMarker: LogMarker, instance: Instance): Future[S3Object] =
-    storeImage(quarantineBucket, ImageIngestOperations.fileKeyFromId(id), file, mimeType, meta, overwrite = true)
+    storeImage(quarantineBucket, ImageIngestOperations.fileKeyFromId(id), file, mimeType, meta, overwrite = true, s3Endpoint = s3Endpoint)
 }
 
 
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala
index 6312450b97..5cd3cc446f 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala
@@ -36,7 +36,7 @@ trait ImageStorage {
     * The file can safely be deleted afterwards.
     */
   def storeImage(bucket: String, id: String, file: File, mimeType: Option[MimeType],
-                 meta: Map[String, String] = Map.empty, overwrite: Boolean)
+                 meta: Map[String, String] = Map.empty, overwrite: Boolean, s3Endpoint: String)
                 (implicit logMarker: LogMarker): Future[S3Object]
 
   def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker): Future[Unit]
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
index 1c7298a3d3..919e688c1c 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
@@ -15,13 +15,13 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage
 
   private val cacheSetting = Some(cacheForever)
   def storeImage(bucket: String, id: String, file: File, mimeType: Option[MimeType],
-                 meta: Map[String, String] = Map.empty, overwrite: Boolean)
+                 meta: Map[String, String] = Map.empty, overwrite: Boolean, s3Endpoint: String)
                 (implicit logMarker: LogMarker) = {
     logger.info(logMarker, s"bucket: $bucket, id: $id, meta: $meta")
     val eventualObject = if (overwrite) {
-      store(bucket, id, file, mimeType, meta, cacheSetting)
+      store(bucket, id, file, mimeType, meta, cacheSetting, s3Endpoint)
     } else {
-      storeIfNotPresent(bucket, id, file, mimeType, meta, cacheSetting)
+      storeIfNotPresent(bucket, id, file, mimeType, meta, cacheSetting, s3Endpoint)
     }
     eventualObject.onComplete(o => logger.info(logMarker, s"storeImage completed $o"))
     eventualObject
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala
index ba98bcd177..0944203002 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala
@@ -6,8 +6,8 @@ import com.gu.mediaservice.model.Instance
 
 import scala.concurrent.ExecutionContext
 
-class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionContext)
-  extends BaseStore[String, ApiAccessor](bucket, config)(ec) {
+class KeyStore(bucket: String, config: CommonConfig, val s3Endpoint: String)(implicit ec: ExecutionContext)
+  extends BaseStore[String, ApiAccessor](bucket, config, s3Endpoint)(ec) {
 
   def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = store.get().get(instance.id + "/" + key)
 
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index f4d1d02ce0..6ddae64a87 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -66,7 +66,6 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
   type Key = String
   type UserMetadata = Map[String, String]
 
-  private val s3Endpoint: String = config.s3Endpoint
   private lazy val client: AmazonS3 = S3Ops.buildS3Client(config)
   // also create a legacy client that uses v2 signatures for URL signing
   private lazy val legacySigningClient: AmazonS3 = S3Ops.buildS3Client(config, forceV2Sigs = true)
@@ -117,7 +116,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     client.doesObjectExist(bucket, key)
   }
 
-  def getObject(bucket: Bucket, url: URI): model.S3Object = {
+  def getObject(bucket: Bucket, url: URI): model.S3Object = { // TODO why can't this just be by bucket + key to remove end point knowledge
     // get path and remove leading `/`
     val key: Key = url.getPath.drop(1)
     client.getObject(new GetObjectRequest(bucket, key))
@@ -166,7 +165,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     client.putObject(bucket, key, content)
   }
 
-  def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None)
+  def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None, s3Endpoint: String)
            (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] =
     Future {
       val metadata = new ObjectMetadata
@@ -190,7 +189,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
       }(markers)
     }
 
-  def storeIfNotPresent(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None)
+  def storeIfNotPresent(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None, s3Endpoint: String)
                        (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = {
     Future{
       Some(client.getObjectMetadata(bucket, id))
@@ -200,13 +199,13 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     }.flatMap {
       case Some(objectMetadata) =>
         logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket $bucket")
-        Future.successful(S3Object(bucket, id, objectMetadata.getContentLength, S3Metadata(objectMetadata) ,s3Endpoint))
+        Future.successful(S3Object(bucket, id, objectMetadata.getContentLength, S3Metadata(objectMetadata), s3Endpoint))
       case None =>
-        store(bucket, id, file, mimeType, meta, cacheControl)
+        store(bucket, id, file, mimeType, meta, cacheControl, s3Endpoint)
     }
   }
 
-  def list(bucket: Bucket, prefixDir: String)
+  def list(bucket: Bucket, prefixDir: String, s3Endpoint: String)
           (implicit ex: ExecutionContext): Future[List[S3Object]] =
     Future {
       val req = new ListObjectsRequest().withBucketName(bucket).withPrefix(s"$prefixDir/")
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
index 4b1e3130de..8a3e584a7e 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
@@ -58,8 +58,6 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
 
   val maybeUploadLimitInBytes: Option[Int] = intOpt("upload.limit.mb").map(_ * 1024 * 1024)
 
-  val s3Endpoint: String= stringOpt("s3.serviceEndpoint").getOrElse("s3.amazonaws.com")
-
   val instancesEndpoint: String = string("instance.service.instances")
 
   // Note: had to make these lazy to avoid init order problems ;_;
@@ -71,6 +69,11 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
 
   val services = new SingleHostServices(domainRoot)
 
+  val imageBucket: String = string("s3.image.bucket")
+  val imageBucketS3Endpoint: String = "s3.amazonaws.com"
+  val thumbnailBucket: String = string("s3.thumb.bucket")
+  val thumbnailBucketS3Endpoint: String = "s3.amazonaws.com"
+
   /**
    * Load in a list of domain metadata specifications from configuration. For example:
    * {{{
diff --git a/common-lib/src/test/resources/application.conf b/common-lib/src/test/resources/application.conf
index 05d1db98e6..e3724393da 100644
--- a/common-lib/src/test/resources/application.conf
+++ b/common-lib/src/test/resources/application.conf
@@ -61,3 +61,7 @@ instance.service.my=""
 instance.service.instances=""
 
 usageEvents.queue.name=""
+
+s3.image.bucket="images"
+
+s3.thumb.bucket="thumbs"
diff --git a/cropper/app/lib/CropStore.scala b/cropper/app/lib/CropStore.scala
index 1cc815d2cb..cf6eb37e6c 100644
--- a/cropper/app/lib/CropStore.scala
+++ b/cropper/app/lib/CropStore.scala
@@ -15,7 +15,7 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS
 
   def storeCropSizing(file: File, filename: String, mimeType: MimeType, crop: Crop, dimensions: Dimensions)(implicit logMarker: LogMarker): Future[Asset] = {
     val metadata = metadataForCrop(crop, dimensions)
-    storeImage(config.imgPublishingBucket, filename, file, Some(mimeType), metadata, overwrite = true) map { s3Object =>
+    storeImage(config.imgPublishingBucket, filename, file, Some(mimeType), metadata, overwrite = true, config.imgPublishingBucketS3Endpoint) map { s3Object =>
       Asset(
         translateImgHost(s3Object.uri),
         Some(s3Object.size),
@@ -27,7 +27,7 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS
   }
 
   def listCrops(id: String, instance: Instance): Future[List[Crop]] = {
-    list(config.imgPublishingBucket, folderForImagesCrops(id, instance)).map { crops => // TODO crops layout want to be pull up
+    list(config.imgPublishingBucket, folderForImagesCrops(id, instance), config.imgPublishingBucketS3Endpoint).map { crops => // TODO crops layout want to be pull up
       crops.foldLeft(Map[String, Crop]()) {
         case (map, (s3Object)) => {
           val filename::containingFolder::_ = s3Object.uri.getPath.split("/").reverse.toList
diff --git a/cropper/app/lib/CropperConfig.scala b/cropper/app/lib/CropperConfig.scala
index 96d043d60a..8aebea551e 100644
--- a/cropper/app/lib/CropperConfig.scala
+++ b/cropper/app/lib/CropperConfig.scala
@@ -7,9 +7,8 @@ import java.io.File
 
 
 class CropperConfig(resources: GridConfigResources) extends CommonConfig(resources) {
-  val imageBucket: String = string("s3.image.bucket")
-
   val imgPublishingBucket = string("publishing.image.bucket")
+  val imgPublishingBucketS3Endpoint: String = "s3.amazonaws.com"
 
   val canDownloadCrop: Boolean = boolean("canDownloadCrop")
 
diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala
index fb40a8889f..f5d7217d5d 100644
--- a/image-loader/app/lib/ImageLoaderConfig.scala
+++ b/image-loader/app/lib/ImageLoaderConfig.scala
@@ -10,12 +10,11 @@ import play.api.inject.ApplicationLifecycle
 import scala.concurrent.duration.FiniteDuration
 
 class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(resources) with StrictLogging {
-  val imageBucket: String = string("s3.image.bucket")
 
   val maybeImageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket")
 
-  val thumbnailBucket: String = string("s3.thumb.bucket")
   val quarantineBucket: Option[String] = stringOpt("s3.quarantine.bucket")
+  val quarantineBucketS3Endpoint: String = "s3.amazonaws.com"
   val uploadToQuarantineEnabled: Boolean = boolean("upload.quarantine.enabled")
 
   val tempDir: File = new File(stringDefault("upload.tmp.dir", "/tmp"))
@@ -66,4 +65,5 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res
       .get[Seq[ImageProcessor]]("image.processors")(configLoader)
     ImageProcessor.compose("ImageConfigLoader-imageProcessor", processors:_*)
   }
+
 }
diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala
index 4a04b04156..290b87c589 100644
--- a/image-loader/app/lib/ImageLoaderStore.scala
+++ b/image-loader/app/lib/ImageLoaderStore.scala
@@ -12,7 +12,7 @@ import java.util.Date
 
 class S3FileDoesNotExistException extends Exception()
 
-class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config) with GridLogging {
+class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config, imageBucketS3Endpoint = config.imageBucketS3Endpoint, thumbnailBucketS3Endpoint = config.thumbnailBucketS3Endpoint) with GridLogging {
 
   private def handleNotFound[T](key: String)(doWork: => T)(loggingIfNotFound: => Unit): T = {
     try {
diff --git a/image-loader/app/lib/QuarantineStore.scala b/image-loader/app/lib/QuarantineStore.scala
index 1c750bb7c1..e73e28ead2 100644
--- a/image-loader/app/lib/QuarantineStore.scala
+++ b/image-loader/app/lib/QuarantineStore.scala
@@ -3,4 +3,4 @@ package lib.storage
 import lib.ImageLoaderConfig
 import com.gu.mediaservice.lib
 
-class QuarantineStore(config: ImageLoaderConfig) extends lib.ImageQuarantineOperations(config.quarantineBucket.get, config)
\ No newline at end of file
+class QuarantineStore(config: ImageLoaderConfig) extends lib.ImageQuarantineOperations(config.quarantineBucket.get, config, s3Endpoint = config.quarantineBucketS3Endpoint)
diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala
index fcfda68a9e..e5961dbfee 100644
--- a/image-loader/app/model/Uploader.scala
+++ b/image-loader/app/model/Uploader.scala
@@ -99,7 +99,7 @@ object Uploader extends GridLogging {
       config.transcodedMimeTypes,
       config.imageBucket,
       config.thumbnailBucket,
-      config.s3Endpoint
+      config.thumbnailBucketS3Endpoint
     )
   }
 
diff --git a/media-api/app/lib/ImageResponse.scala b/media-api/app/lib/ImageResponse.scala
index c96aaade7c..09cfeb1965 100644
--- a/media-api/app/lib/ImageResponse.scala
+++ b/media-api/app/lib/ImageResponse.scala
@@ -79,7 +79,7 @@ class ImageResponse(config: MediaApiConfig, s3Client: S3Client, usageQuota: Usag
     val pngUrl: Option[String] = pngFileUri
       .map(s3Client.signUrl(config.imageBucket, _, image, imageType = OptimisedPng))
 
-    def s3SignedThumbUrl = s3Client.signUrl(config.thumbBucket, fileUri, image, imageType = Thumbnail)
+    def s3SignedThumbUrl = s3Client.signUrl(config.thumbnailBucket, fileUri, image, imageType = Thumbnail)
 
     val thumbUrl = config.cloudFrontDomainThumbBucket
       .flatMap(s3Client.signedCloudFrontUrl(_, fileUri.getPath.drop(1)))
diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala
index 081a9f347b..7941496fba 100644
--- a/media-api/app/lib/MediaApiConfig.scala
+++ b/media-api/app/lib/MediaApiConfig.scala
@@ -11,7 +11,8 @@ import scala.util.Try
 
 case class StoreConfig(
   storeBucket: String,
-  storeKey: String
+  storeKey: String,
+  storeBucketS3Endpoint: String
 )
 
 class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) {
@@ -19,14 +20,11 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla
   val usageMailBucket: String = string("s3.usagemail.bucket")
 
   val quotaStoreKey: String = string("quota.store.key")
-  val quotaStoreConfig: StoreConfig = StoreConfig(configBucket, quotaStoreKey)
+  val quotaStoreConfig: StoreConfig = StoreConfig(configBucket, quotaStoreKey, "s3.amazonaws.com")
 
   //Lazy allows this to be empty and not break things unless used somewhere
   lazy val imgPublishingBucket = string("publishing.image.bucket")
 
-  val imageBucket: String = string("s3.image.bucket")
-  val thumbBucket: String = string("s3.thumb.bucket")
-
   val cloudFrontDomainThumbBucket: Option[String]   = stringOpt("cloudfront.domain.thumbbucket")
   val cloudFrontPrivateKeyBucket: Option[String]    = stringOpt("cloudfront.private-key.bucket")
   val cloudFrontPrivateKeyBucketKey: Option[String] = stringOpt("cloudfront.private-key.key")
diff --git a/media-api/app/lib/QuotaStore.scala b/media-api/app/lib/QuotaStore.scala
index c79486fbd1..ce1e3f8f5c 100644
--- a/media-api/app/lib/QuotaStore.scala
+++ b/media-api/app/lib/QuotaStore.scala
@@ -8,8 +8,9 @@ import scala.concurrent.ExecutionContext
 class QuotaStore(
                   quotaFile: String,
                   bucket: String,
-                  config: MediaApiConfig
-                )(implicit ec: ExecutionContext) extends BaseStore[String, SupplierUsageQuota](bucket, config)(ec) {
+                  config: MediaApiConfig,
+                  val s3Endpoint: String,
+                )(implicit ec: ExecutionContext) extends BaseStore[String, SupplierUsageQuota](bucket, config, s3Endpoint)(ec) {
 
   def getQuota: Map[String, SupplierUsageQuota] = store.get()
 
diff --git a/media-api/app/lib/UsageQuota.scala b/media-api/app/lib/UsageQuota.scala
index bcd4e93e21..c508085fc4 100644
--- a/media-api/app/lib/UsageQuota.scala
+++ b/media-api/app/lib/UsageQuota.scala
@@ -16,13 +16,15 @@ class UsageQuota(config: MediaApiConfig, scheduler: Scheduler) {
   val quotaStore = new QuotaStore(
     config.quotaStoreConfig.storeKey,
     config.quotaStoreConfig.storeBucket,
-    config
+    config,
+    config.quotaStoreConfig.storeBucketS3Endpoint
   )
 
   val usageStore = new UsageStore(
     config.usageMailBucket,
     config,
-    quotaStore
+    quotaStore,
+    "s3.amazonaws.com"
   )
 
   def scheduleUpdates(): Unit = {
diff --git a/media-api/app/lib/UsageStore.scala b/media-api/app/lib/UsageStore.scala
index 2c19044131..a2f3e8130b 100644
--- a/media-api/app/lib/UsageStore.scala
+++ b/media-api/app/lib/UsageStore.scala
@@ -59,8 +59,9 @@ object UsageStore extends GridLogging {
 class UsageStore(
   bucket: String,
   config: MediaApiConfig,
-  quotaStore: QuotaStore
-)(implicit val ec: ExecutionContext) extends BaseStore[String, UsageStatus](bucket, config) with GridLogging {
+  quotaStore: QuotaStore,
+  val s3Endpoint: String
+)(implicit val ec: ExecutionContext) extends BaseStore[String, UsageStatus](bucket, config, s3Endpoint) with GridLogging {
   import UsageStore._
 
   def getUsageStatusForUsageRights(usageRights: UsageRights): Future[UsageStatus] = {
diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala
index f264d8589f..f7dc196512 100644
--- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala
+++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala
@@ -25,7 +25,7 @@ class ApiKeyAuthenticationProvider(configuration: Configuration, resources: Auth
   var keyStorePlaceholder: Option[KeyStore] = _
 
   override def initialise(): Unit = {
-    val store = new KeyStore(configuration.get[String]("authKeyStoreBucket"), resources.commonConfig)
+    val store = new KeyStore(configuration.get[String]("authKeyStoreBucket"), resources.commonConfig, "s3.amazonaws.com")
     store.scheduleUpdates(resources.actorSystem.scheduler)
     keyStorePlaceholder = Some(store)
   }
diff --git a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala
index 3ba9231943..c5308a584e 100644
--- a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala
+++ b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala
@@ -43,7 +43,7 @@ class ApiKeyAuthenticationProviderTest extends AsyncFreeSpec with Matchers with
       Future.successful(())
     }
 
-    override def keyStore: KeyStore = new KeyStore("not-used", resources.commonConfig) {
+    override def keyStore: KeyStore = new KeyStore("not-used", resources.commonConfig, "s3.amazonaws.com") {
       override def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = {
         key match {
           case "key-chuckle" => Some(ApiAccessor("brothers", Internal))
diff --git a/scripts/src/main/scala/com/gu/mediaservice/lib/JsonValueCodecJsValue.scala b/scripts/src/main/scala/com/gu/mediaservice/lib/JsonValueCodecJsValue.scala
index da73d03db6..c6ad7b03f8 100644
--- a/scripts/src/main/scala/com/gu/mediaservice/lib/JsonValueCodecJsValue.scala
+++ b/scripts/src/main/scala/com/gu/mediaservice/lib/JsonValueCodecJsValue.scala
@@ -98,6 +98,7 @@ object JsonValueCodecJsValue {
           case JsNull =>
             out.writeNull()
           case _ =>
+            out.writeNull()
         }
       }
 
diff --git a/thrall/app/lib/ThrallConfig.scala b/thrall/app/lib/ThrallConfig.scala
index 25743ddf67..7b4d3b03ae 100644
--- a/thrall/app/lib/ThrallConfig.scala
+++ b/thrall/app/lib/ThrallConfig.scala
@@ -56,10 +56,6 @@ object KinesisReceiverConfig {
 }
 
 class ThrallConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) {
-  val imageBucket: String = string("s3.image.bucket")
-
-  val thumbnailBucket: String = string("s3.thumb.bucket")
-
   val maybeReaperBucket: Option[String] = stringOpt("s3.reaper.bucket")
   val maybeReaperCountPerRun: Option[Int] = intOpt("reaper.countPerRun")
 
diff --git a/thrall/app/lib/ThrallStore.scala b/thrall/app/lib/ThrallStore.scala
index a221c0471c..4deb3e5444 100644
--- a/thrall/app/lib/ThrallStore.scala
+++ b/thrall/app/lib/ThrallStore.scala
@@ -2,4 +2,4 @@ package lib
 
 import com.gu.mediaservice.lib
 
-class ThrallStore(config: ThrallConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config, config.isVersionedS3)
+class ThrallStore(config: ThrallConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config, config.isVersionedS3, config.imageBucketS3Endpoint, config.thumbnailBucketS3Endpoint)

From b923efa893315409eb22adbdbed4a5366f38eaeb Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 29 Nov 2024 16:33:06 +0000
Subject: [PATCH 04/45] Setting up for GCP and AWS clients by introducing s3
 end point along side bucket.

Bucket should be (bucket, endPoint, client?)
---
 .../com/gu/mediaservice/lib/BaseStore.scala   |   6 +-
 .../lib/ImageIngestOperations.scala           |  30 ++---
 .../lib/ImageQuarantineOperations.scala       |   6 +-
 .../gu/mediaservice/lib/ImageStorage.scala    |   9 +-
 .../gu/mediaservice/lib/S3ImageStorage.scala  |  18 +--
 .../gu/mediaservice/lib/auth/KeyStore.scala   |   6 +-
 .../com/gu/mediaservice/lib/aws/S3.scala      | 123 ++++++++++--------
 .../gu/mediaservice/lib/aws/S3Bucket.scala    |   3 +
 .../lib/config/CommonConfig.scala             |  13 +-
 cropper/app/lib/CropStore.scala               |   6 +-
 cropper/app/lib/CropperConfig.scala           |   8 +-
 cropper/app/lib/Crops.scala                   |   4 +-
 cropper/test/lib/CropsTest.scala              |   3 +-
 image-loader/app/lib/ImageLoaderConfig.scala  |   7 +-
 image-loader/app/lib/ImageLoaderStore.scala   |   8 +-
 image-loader/app/lib/QuarantineStore.scala    |   2 +-
 image-loader/app/model/Projector.scala        |  20 +--
 image-loader/app/model/Uploader.scala         |   8 +-
 .../test/scala/model/ImageUploadTest.scala    |   6 +-
 .../test/scala/model/ProjectorTest.scala      |   3 +-
 media-api/app/controllers/MediaApi.scala      |   2 +-
 media-api/app/lib/MediaApiConfig.scala        |  16 +--
 media-api/app/lib/QuotaStore.scala            |   6 +-
 media-api/app/lib/UsageQuota.scala            |   3 +-
 media-api/app/lib/UsageStore.scala            |   6 +-
 .../ApiKeyAuthenticationProvider.scala        |   4 +-
 .../ApiKeyAuthenticationProviderTest.scala    |   3 +-
 thrall/app/controllers/ThrallController.scala |   6 +-
 thrall/app/lib/ThrallConfig.scala             |   4 +-
 thrall/app/lib/ThrallStore.scala              |   2 +-
 30 files changed, 179 insertions(+), 162 deletions(-)
 create mode 100644 common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala
index 31d3cbb610..5a0f2eb36d 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala
@@ -1,7 +1,7 @@
 package com.gu.mediaservice.lib
 
 import org.apache.pekko.actor.{Cancellable, Scheduler}
-import com.gu.mediaservice.lib.aws.S3
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
 import com.gu.mediaservice.lib.config.CommonConfig
 import com.gu.mediaservice.lib.logging.GridLogging
 import org.joda.time.DateTime
@@ -14,11 +14,9 @@ import scala.concurrent.duration._
 import scala.util.control.NonFatal
 
 
-abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonConfig, s3Endpoint: String)(implicit ec: ExecutionContext)
+abstract class BaseStore[TStoreKey, TStoreVal](bucket: S3Bucket, config: CommonConfig)(implicit ec: ExecutionContext)
   extends GridLogging {
 
-  def s3Endpoint: String
-
   val s3 = new S3(config)
 
   protected val store: AtomicReference[Map[TStoreKey, TStoreVal]] = new AtomicReference(Map.empty)
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index fb8e797204..f1e6a0f26a 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -4,7 +4,7 @@ import com.amazonaws.services.s3.model.MultiObjectDeleteException
 
 import java.io.File
 import com.gu.mediaservice.lib.config.CommonConfig
-import com.gu.mediaservice.lib.aws.S3Object
+import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object}
 import com.gu.mediaservice.lib.logging.LogMarker
 import com.gu.mediaservice.model.{Instance, MimeType, Png}
 import com.typesafe.scalalogging.StrictLogging
@@ -21,7 +21,7 @@ object ImageIngestOperations {
   private def snippetForId(id: String) = id.take(6).mkString("/") + "/" + id
 }
 
-class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config: CommonConfig, isVersionedS3: Boolean = false, imageBucketS3Endpoint: String, thumbnailBucketS3Endpoint: String)
+class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, config: CommonConfig, isVersionedS3: Boolean = false)
   extends S3ImageStorage(config) with StrictLogging {
 
   import ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId}
@@ -38,7 +38,7 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
     val instanceSpecificKey = instanceAwareOriginalImageKey(storableImage)
     logger.info(s"Storing original image to instance specific key:$imageBucket / $instanceSpecificKey")
     storeImage(imageBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType),
-      storableImage.meta, overwrite = false, s3Endpoint = imageBucketS3Endpoint)
+      storableImage.meta, overwrite = false)
   }
 
   private def storeThumbnailImage(storableImage: StorableThumbImage)
@@ -46,7 +46,7 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
     val instanceSpecificKey = instanceAwareThumbnailImageKey(storableImage)
     logger.info(s"Storing thumbnail to instance specific key: $thumbnailBucket / $instanceSpecificKey")
     storeImage(thumbnailBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType),
-      overwrite = true, s3Endpoint = thumbnailBucketS3Endpoint)
+      overwrite = true)
   }
 
   private def storeOptimisedImage(storableImage: StorableOptimisedImage)
@@ -54,10 +54,10 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
     val instanceSpecificKey = optimisedPngKeyFromId(storableImage.id)(storableImage.instance)
     logger.info(s"Storing optimised image to instance specific key: $thumbnailBucket / $instanceSpecificKey")
     storeImage(imageBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType),
-      overwrite = true, s3Endpoint = imageBucketS3Endpoint)
+      overwrite = true)
   }
 
-  private def bulkDelete(bucket: String, keys: List[String]): Future[Map[String, Boolean]] = keys match {
+  private def bulkDelete(bucket: S3Bucket, keys: List[String]): Future[Map[String, Boolean]] = keys match {
     case Nil => Future.successful(Map.empty)
     case _ => Future {
       try {
@@ -105,38 +105,38 @@ sealed trait ImageWrapper {
   val instance: Instance
 }
 sealed trait StorableImage extends ImageWrapper {
-  def toProjectedS3Object(thumbBucket: String, s3Endpoint: String): S3Object = S3Object(
-    thumbBucket,
+  def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object(
+    thumbBucket.bucket,
     ImageIngestOperations.fileKeyFromId(id)(instance),
     file,
     Some(mimeType),
     lastModified = None,
     meta,
-    s3Endpoint = s3Endpoint
+    s3Endpoint = thumbBucket.endpoint
   )
 }
 
 case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage
 case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: DateTime, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage {
-  override def toProjectedS3Object(thumbBucket: String, s3Endpoint: String): S3Object = S3Object(
-    thumbBucket,
+  override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object(
+    thumbBucket.bucket,
     ImageIngestOperations.fileKeyFromId(id)(instance),
     file,
     Some(mimeType),
     lastModified = Some(lastModified),
     meta,
-    s3Endpoint = s3Endpoint
+    s3Endpoint = thumbBucket.endpoint
   )
 }
 case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage {
-  override def toProjectedS3Object(thumbBucket: String, s3Endpoint: String): S3Object = S3Object(
-    thumbBucket,
+  override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object(
+    thumbBucket.bucket,
     ImageIngestOperations.optimisedPngKeyFromId(id)(instance),
     file,
     Some(mimeType),
     lastModified = None,
     meta = meta,
-    s3Endpoint = s3Endpoint
+    s3Endpoint = thumbBucket.endpoint
   )
 }
 
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala
index 51b04e980d..eaf6b82979 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala
@@ -2,18 +2,18 @@ package com.gu.mediaservice.lib
 
 import java.io.File
 import com.gu.mediaservice.lib.config.CommonConfig
-import com.gu.mediaservice.lib.aws.S3Object
+import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object}
 import com.gu.mediaservice.lib.logging.LogMarker
 import com.gu.mediaservice.model.{Instance, MimeType}
 
 import scala.concurrent.Future
 
-class ImageQuarantineOperations(quarantineBucket: String, config: CommonConfig, isVersionedS3: Boolean = false, s3Endpoint: String)
+class ImageQuarantineOperations(quarantineBucket: S3Bucket, config: CommonConfig, isVersionedS3: Boolean = false)
   extends S3ImageStorage(config) {
 
   def storeQuarantineImage(id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty)
                        (implicit logMarker: LogMarker, instance: Instance): Future[S3Object] =
-    storeImage(quarantineBucket, ImageIngestOperations.fileKeyFromId(id), file, mimeType, meta, overwrite = true, s3Endpoint = s3Endpoint)
+    storeImage(quarantineBucket, ImageIngestOperations.fileKeyFromId(id), file, mimeType, meta, overwrite = true)
 }
 
 
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala
index 5cd3cc446f..a6c45c346c 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala
@@ -2,11 +2,10 @@ package com.gu.mediaservice.lib
 
 import java.util.concurrent.Executors
 import java.io.File
-
 import scala.concurrent.{ExecutionContext, Future}
 import scala.concurrent.duration._
 import scala.language.postfixOps
-import com.gu.mediaservice.lib.aws.S3Object
+import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object}
 import com.gu.mediaservice.lib.logging.LogMarker
 import com.gu.mediaservice.model.MimeType
 
@@ -35,9 +34,9 @@ trait ImageStorage {
   /** Store a copy of the given file and return the URI of that copy.
     * The file can safely be deleted afterwards.
     */
-  def storeImage(bucket: String, id: String, file: File, mimeType: Option[MimeType],
-                 meta: Map[String, String] = Map.empty, overwrite: Boolean, s3Endpoint: String)
+  def storeImage(bucket: S3Bucket, id: String, file: File, mimeType: Option[MimeType],
+                 meta: Map[String, String] = Map.empty, overwrite: Boolean)
                 (implicit logMarker: LogMarker): Future[S3Object]
 
-  def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker): Future[Unit]
+  def deleteImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker): Future[Unit]
 }
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
index 919e688c1c..54b17dba01 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
@@ -1,6 +1,6 @@
 package com.gu.mediaservice.lib
 
-import com.gu.mediaservice.lib.aws.S3
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
 import com.gu.mediaservice.lib.config.CommonConfig
 import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker}
 import com.gu.mediaservice.model.MimeType
@@ -14,32 +14,32 @@ import scala.concurrent.Future
 class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage with GridLogging {
 
   private val cacheSetting = Some(cacheForever)
-  def storeImage(bucket: String, id: String, file: File, mimeType: Option[MimeType],
-                 meta: Map[String, String] = Map.empty, overwrite: Boolean, s3Endpoint: String)
+  def storeImage(bucket: S3Bucket, id: String, file: File, mimeType: Option[MimeType],
+                 meta: Map[String, String] = Map.empty, overwrite: Boolean)
                 (implicit logMarker: LogMarker) = {
     logger.info(logMarker, s"bucket: $bucket, id: $id, meta: $meta")
     val eventualObject = if (overwrite) {
-      store(bucket, id, file, mimeType, meta, cacheSetting, s3Endpoint)
+      store(bucket, id, file, mimeType, meta, cacheSetting)
     } else {
-      storeIfNotPresent(bucket, id, file, mimeType, meta, cacheSetting, s3Endpoint)
+      storeIfNotPresent(bucket, id, file, mimeType, meta, cacheSetting)
     }
     eventualObject.onComplete(o => logger.info(logMarker, s"storeImage completed $o"))
     eventualObject
   }
 
-  def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future {
+  def deleteImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker) = Future {
     deleteObject(bucket, id)
     logger.info(logMarker, s"Deleted image $id from bucket $bucket")
   }
 
-  def deleteVersionedImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future {
+  def deleteVersionedImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker) = Future {
     val objectVersion = getObjectMetadata(bucket, id).getVersionId
     deleteVersion(bucket, id, objectVersion)
     logger.info(logMarker, s"Deleted image $id from bucket $bucket (version: $objectVersion)")
   }
 
-  def deleteFolder(bucket: String, id: String)(implicit logMarker: LogMarker) = Future {
-		val files = listObjects(bucket, id).getObjectSummaries.asScala
+  def deleteFolder(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker) = Future {
+    val files = listObjects(bucket, id).getObjectSummaries.asScala
     logger.info(s"Found ${files.size} files to delete in folder $id")
     files.foreach(file => deleteObject(bucket, file.getKey))
 		logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket")
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala
index 0944203002..ebc43bf09c 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala
@@ -1,13 +1,14 @@
 package com.gu.mediaservice.lib.auth
 
 import com.gu.mediaservice.lib.BaseStore
+import com.gu.mediaservice.lib.aws.S3Bucket
 import com.gu.mediaservice.lib.config.CommonConfig
 import com.gu.mediaservice.model.Instance
 
 import scala.concurrent.ExecutionContext
 
-class KeyStore(bucket: String, config: CommonConfig, val s3Endpoint: String)(implicit ec: ExecutionContext)
-  extends BaseStore[String, ApiAccessor](bucket, config, s3Endpoint)(ec) {
+class KeyStore(bucket: S3Bucket, config: CommonConfig)(implicit ec: ExecutionContext)
+  extends BaseStore[String, ApiAccessor](bucket, config)(ec) {
 
   def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = store.get().get(instance.id + "/" + key)
 
@@ -18,5 +19,4 @@ class KeyStore(bucket: String, config: CommonConfig, val s3Endpoint: String)(imp
   private def fetchAll: Map[String, ApiAccessor] = {
     s3.listObjectKeys(bucket).flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap
   }
-
 }
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index 6ddae64a87..ea1c6ba2f6 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -62,15 +62,21 @@ object S3Metadata {
 case class S3ObjectMetadata(contentType: Option[MimeType], cacheControl: Option[String], lastModified: Option[DateTime])
 
 class S3(config: CommonConfig) extends GridLogging with ContentDisposition with RoundedExpiration {
-  type Bucket = String
   type Key = String
   type UserMetadata = Map[String, String]
 
-  private lazy val client: AmazonS3 = S3Ops.buildS3Client(config)
+  val AmazonAwsS3Endpoint: String = S3.AmazonAwsS3Endpoint
+
+  private lazy val amazonS3: AmazonS3 = S3Ops.buildS3Client(config)
   // also create a legacy client that uses v2 signatures for URL signing
   private lazy val legacySigningClient: AmazonS3 = S3Ops.buildS3Client(config, forceV2Sigs = true)
 
-  def signUrl(bucket: Bucket, url: URI, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = {
+  private def clientFor(bucket: S3Bucket): AmazonS3 = {
+    logger.info("Client for: " + bucket.endpoint)
+    amazonS3
+  }
+
+  def signUrl(bucket: S3Bucket, url: URI, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = {
     // get path and remove leading `/`
     val key: Key = url.getPath.drop(1)
 
@@ -78,60 +84,60 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
 
     val headers = new ResponseHeaderOverrides().withContentDisposition(contentDisposition)
 
-    val request = new GeneratePresignedUrlRequest(bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers)
+    val request = new GeneratePresignedUrlRequest(bucket.bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers)
     legacySigningClient.generatePresignedUrl(request).toExternalForm
   }
 
-  def signUrlTony(bucket: Bucket, url: URI, expiration: DateTime = cachableExpiration()): URL = {
+  def signUrlTony(bucket: S3Bucket, url: URI, expiration: DateTime = cachableExpiration()): URL = {
     // get path and remove leading `/`
     val key: Key = url.getPath.drop(1)
 
-    val request = new GeneratePresignedUrlRequest(bucket, key).withExpiration(expiration.toDate)
+    val request = new GeneratePresignedUrlRequest(bucket.bucket, key).withExpiration(expiration.toDate)
     legacySigningClient.generatePresignedUrl(request)
   }
 
-  def copyObject(sourceBucket: Bucket, destinationBucket: Bucket, key: String): CopyObjectResult = {
-    client.copyObject(sourceBucket, key, destinationBucket, key)
+  def copyObject(sourceBucket: S3Bucket, destinationBucket: S3Bucket, key: String): CopyObjectResult = {
+    clientFor(sourceBucket).copyObject(sourceBucket.bucket, key, destinationBucket.bucket, key)
   }
 
-  def generatePresignedRequest(request: GeneratePresignedUrlRequest): URL = {
-    client.generatePresignedUrl(request)
+  def generatePresignedRequest(request: GeneratePresignedUrlRequest, bucket: S3Bucket): URL = {
+    clientFor(bucket).generatePresignedUrl(request)
   }
 
-  def deleteObject(bucket: Bucket, key: String): Unit = {
-    client.deleteObject(bucket, key)
+  def deleteObject(bucket: S3Bucket, key: String): Unit = {
+    clientFor(bucket).deleteObject(bucket.bucket, key)
   }
 
-  def deleteObjects(bucket: Bucket, keys: Seq[Bucket]): DeleteObjectsResult = {
-    client.deleteObjects(
-      new DeleteObjectsRequest(bucket).withKeys(keys: _*)
+  def deleteObjects(bucket: S3Bucket, keys: Seq[String]): DeleteObjectsResult = {
+    clientFor(bucket).deleteObjects(
+      new DeleteObjectsRequest(bucket.bucket).withKeys(keys: _*)
     )
   }
 
-  def deleteVersion(bucket: Bucket, id: Bucket, objectVersion: Bucket): Unit = {
-    client.deleteVersion(bucket, id, objectVersion)
+  def deleteVersion(bucket: S3Bucket, id: String, objectVersion: String): Unit = {
+    clientFor(bucket).deleteVersion(bucket.bucket, id, objectVersion)
   }
 
-  def doesObjectExist(bucket: Bucket, key: String) = {
-    client.doesObjectExist(bucket, key)
+  def doesObjectExist(bucket: S3Bucket, key: String) = {
+    clientFor(bucket).doesObjectExist(bucket.bucket, key)
   }
 
-  def getObject(bucket: Bucket, url: URI): model.S3Object = { // TODO why can't this just be by bucket + key to remove end point knowledge
+  def getObject(bucket: S3Bucket, url: URI): model.S3Object = { // TODO why can't this just be by bucket + key to remove end point knowledge
     // get path and remove leading `/`
     val key: Key = url.getPath.drop(1)
-    client.getObject(new GetObjectRequest(bucket, key))
+    clientFor(bucket).getObject(new GetObjectRequest(bucket.bucket, key))
   }
 
-  def getObject(bucket: Bucket, key: String): model.S3Object = {
-    client.getObject(new GetObjectRequest(bucket, key))
+  def getObject(bucket: S3Bucket, key: String): model.S3Object = {
+    clientFor(bucket).getObject(new GetObjectRequest(bucket.bucket, key))
   }
 
-  def getObject(bucket: Bucket, obj: S3ObjectSummary): model.S3Object = {
-    client.getObject(bucket, obj.getKey)
+  def getObject(bucket: S3Bucket, obj: S3ObjectSummary): model.S3Object = {
+    clientFor(bucket).getObject(bucket.bucket, obj.getKey)
   }
 
-  def getObjectAsString(bucket: Bucket, key: String): Option[String] = {
-    val content = client.getObject(new GetObjectRequest(bucket, key))
+  def getObjectAsString(bucket: S3Bucket, key: String): Option[String] = {
+    val content = clientFor(bucket).getObject(new GetObjectRequest(bucket.bucket, key))
     val stream = content.getObjectContent
     try {
       Some(IOUtils.toString(stream).trim)
@@ -145,27 +151,27 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     }
   }
 
-  def getObjectMetadata(bucket: Bucket, id: Bucket): ObjectMetadata = {
-    client.getObjectMetadata(bucket, id)
+  def getObjectMetadata(bucket: S3Bucket, id: String): ObjectMetadata = {
+    clientFor(bucket).getObjectMetadata(bucket.bucket, id)
   }
 
-  def listObjects(bucket: String): ObjectListing = {
-    client.listObjects(bucket)
+  def listObjects(bucket: S3Bucket): ObjectListing = {
+    clientFor(bucket).listObjects(bucket.bucket)
   }
 
-  def listObjects(bucket: String, prefix: String): ObjectListing = {
-    client.listObjects(bucket, prefix)
+  def listObjects(bucket: S3Bucket, prefix: String): ObjectListing = {
+    clientFor(bucket).listObjects(bucket.bucket, prefix)
   }
 
-  def listObjectKeys(bucket: String): Seq[String] = {
-    client.listObjects(bucket).getObjectSummaries.asScala.map(_.getKey).toSeq
+  def listObjectKeys(bucket: S3Bucket): Seq[String] = {
+    clientFor(bucket).listObjects(bucket.bucket).getObjectSummaries.asScala.map(_.getKey).toSeq
   }
 
-  def putObject(bucket: String, key: String, content: String): Unit = {
-    client.putObject(bucket, key, content)
+  def putObject(bucket: S3Bucket, key: String, content: String): Unit = {
+    clientFor(bucket).putObject(bucket.bucket, key, content)
   }
 
-  def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None, s3Endpoint: String)
+  def store(bucket: S3Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None)
            (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] =
     Future {
       val metadata = new ObjectMetadata
@@ -180,54 +186,55 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
       )
       val markers = logMarker ++ fileMarkers
 
-      val req = new PutObjectRequest(bucket, id, file).withMetadata(metadata)
+      val req = new PutObjectRequest(bucket.bucket, id, file).withMetadata(metadata)
       Stopwatch(s"S3 client.putObject ($req)"){
+        val client = clientFor(bucket)
         client.putObject(req)
         // once we've completed the PUT read back to ensure that we are returning reality
-        val metadata = client.getObjectMetadata(bucket, id)
-        S3Object(bucket, id, metadata.getContentLength, S3Metadata(metadata), s3Endpoint)
+        val metadata = client.getObjectMetadata(bucket.bucket, id)
+        S3Object(bucket.bucket, id, metadata.getContentLength, S3Metadata(metadata), bucket.endpoint)
       }(markers)
     }
 
-  def storeIfNotPresent(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None, s3Endpoint: String)
+  def storeIfNotPresent(bucket: S3Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None)
                        (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = {
     Future{
-      Some(client.getObjectMetadata(bucket, id))
+      Some(clientFor(bucket).getObjectMetadata(bucket.bucket, id))
     }.recover {
       // translate this exception into the object not existing
       case as3e:AmazonS3Exception if as3e.getStatusCode == 404 => None
     }.flatMap {
       case Some(objectMetadata) =>
         logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket $bucket")
-        Future.successful(S3Object(bucket, id, objectMetadata.getContentLength, S3Metadata(objectMetadata), s3Endpoint))
+        Future.successful(S3Object(bucket.bucket, id, objectMetadata.getContentLength, S3Metadata(objectMetadata), bucket.endpoint))
       case None =>
-        store(bucket, id, file, mimeType, meta, cacheControl, s3Endpoint)
+        store(bucket, id, file, mimeType, meta, cacheControl)
     }
   }
 
-  def list(bucket: Bucket, prefixDir: String, s3Endpoint: String)
+  def list(bucket: S3Bucket, prefixDir: String)
           (implicit ex: ExecutionContext): Future[List[S3Object]] =
     Future {
-      val req = new ListObjectsRequest().withBucketName(bucket).withPrefix(s"$prefixDir/")
-      val listing = client.listObjects(req)
+      val req = new ListObjectsRequest().withBucketName(bucket.bucket).withPrefix(s"$prefixDir/")
+      val listing = clientFor(bucket).listObjects(req)
       val summaries = listing.getObjectSummaries.asScala
       summaries.map(summary => (summary.getKey, summary)).foldLeft(List[S3Object]()) {
         case (memo: List[S3Object], (key: String, summary: S3ObjectSummary)) =>
-          S3Object(bucket, key, summary.getSize, getMetadata(bucket, key), s3Endpoint) :: memo
+          S3Object(bucket.bucket, key, summary.getSize, getMetadata(bucket, key), bucket.endpoint) :: memo
       }
     }
 
-  def getMetadata(bucket: Bucket, key: Key): S3Metadata = {
-    val meta = client.getObjectMetadata(bucket, key)
+  def getMetadata(bucket: S3Bucket, key: Key): S3Metadata = {
+    val meta = clientFor(bucket).getObjectMetadata(bucket.bucket, key)
     S3Metadata(meta)
   }
 
-  def getUserMetadata(bucket: Bucket, key: Key): Map[Bucket, Bucket] =
-    client.getObjectMetadata(bucket, key).getUserMetadata.asScala.toMap
+  def getUserMetadata(bucket: S3Bucket, key: Key): Map[String, String] =
+    clientFor(bucket).getObjectMetadata(bucket.bucket, key).getUserMetadata.asScala.toMap
 
-  def syncFindKey(bucket: Bucket, prefixName: String): Option[Key] = {
-    val req = new ListObjectsRequest().withBucketName(bucket).withPrefix(s"$prefixName-")
-    val listing = client.listObjects(req)
+  def syncFindKey(bucket: S3Bucket, prefixName: String): Option[Key] = {
+    val req = new ListObjectsRequest().withBucketName(bucket.bucket).withPrefix(s"$prefixName-")
+    val listing = clientFor(bucket).listObjects(req)
     val summaries = listing.getObjectSummaries.asScala
     summaries.headOption.map(_.getKey)
   }
@@ -256,3 +263,7 @@ object S3Ops {
     config.withAWSCredentials(builder, localstackAware, maybeRegionOverride).build()
   }
 }
+
+object S3 {
+  val AmazonAwsS3Endpoint: String = "s3.amazonaws.com"
+}
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
new file mode 100644
index 0000000000..2fc155c89c
--- /dev/null
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
@@ -0,0 +1,3 @@
+package com.gu.mediaservice.lib.aws
+
+case class S3Bucket(bucket: String, endpoint: String)
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
index 8a3e584a7e..8745bc0710 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
@@ -1,6 +1,6 @@
 package com.gu.mediaservice.lib.config
 
-import com.gu.mediaservice.lib.aws.{AwsClientV1BuilderUtils, AwsClientV2BuilderUtils, KinesisSenderConfig}
+import com.gu.mediaservice.lib.aws.{AwsClientV1BuilderUtils, AwsClientV2BuilderUtils, KinesisSenderConfig, S3, S3Bucket}
 import com.gu.mediaservice.model.UsageRightsSpec
 import com.typesafe.config.Config
 import com.typesafe.scalalogging.StrictLogging
@@ -53,8 +53,8 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
   lazy val softDeletedMetadataTable: String = string("dynamo.table.softDelete.metadata")
 
   val maybeIngestSqsQueueUrl: Option[String] = stringOpt("sqs.ingest.queue.url")
-  val maybeIngestBucket: Option[String] = stringOpt("s3.ingest.bucket")
-  val maybeFailBucket: Option[String] = stringOpt("s3.fail.bucket")
+  val maybeIngestBucket: Option[S3Bucket] = stringOpt("s3.ingest.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint))
+  val maybeFailBucket: Option[S3Bucket] = stringOpt("s3.fail.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint))
 
   val maybeUploadLimitInBytes: Option[Int] = intOpt("upload.limit.mb").map(_ * 1024 * 1024)
 
@@ -69,10 +69,9 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
 
   val services = new SingleHostServices(domainRoot)
 
-  val imageBucket: String = string("s3.image.bucket")
-  val imageBucketS3Endpoint: String = "s3.amazonaws.com"
-  val thumbnailBucket: String = string("s3.thumb.bucket")
-  val thumbnailBucketS3Endpoint: String = "s3.amazonaws.com"
+  val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket"), S3.AmazonAwsS3Endpoint)
+  val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket"), S3.AmazonAwsS3Endpoint)
+  val thumbnailBucketS3Endpoint: String = S3.AmazonAwsS3Endpoint
 
   /**
    * Load in a list of domain metadata specifications from configuration. For example:
diff --git a/cropper/app/lib/CropStore.scala b/cropper/app/lib/CropStore.scala
index cf6eb37e6c..60b3a2672e 100644
--- a/cropper/app/lib/CropStore.scala
+++ b/cropper/app/lib/CropStore.scala
@@ -15,7 +15,7 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS
 
   def storeCropSizing(file: File, filename: String, mimeType: MimeType, crop: Crop, dimensions: Dimensions)(implicit logMarker: LogMarker): Future[Asset] = {
     val metadata = metadataForCrop(crop, dimensions)
-    storeImage(config.imgPublishingBucket, filename, file, Some(mimeType), metadata, overwrite = true, config.imgPublishingBucketS3Endpoint) map { s3Object =>
+    storeImage(config.imgPublishingBucket, filename, file, Some(mimeType), metadata, overwrite = true) map { s3Object =>
       Asset(
         translateImgHost(s3Object.uri),
         Some(s3Object.size),
@@ -27,7 +27,7 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS
   }
 
   def listCrops(id: String, instance: Instance): Future[List[Crop]] = {
-    list(config.imgPublishingBucket, folderForImagesCrops(id, instance), config.imgPublishingBucketS3Endpoint).map { crops => // TODO crops layout want to be pull up
+    list(config.imgPublishingBucket, folderForImagesCrops(id, instance)).map { crops => // TODO crops layout want to be pull up
       crops.foldLeft(Map[String, Crop]()) {
         case (map, (s3Object)) => {
           val filename::containingFolder::_ = s3Object.uri.getPath.split("/").reverse.toList
@@ -78,7 +78,7 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS
   def translateImgHost(uri: URI): URI =
     new URI("https", config.imgPublishingHost, uri.getPath, uri.getFragment)
 
-  private def folderForImagesCrops(id: Bucket, instance: Instance) = {
+  private def folderForImagesCrops(id: String, instance: Instance) = {
     instance.id + "/" + id
   }
 
diff --git a/cropper/app/lib/CropperConfig.scala b/cropper/app/lib/CropperConfig.scala
index 8aebea551e..395122932c 100644
--- a/cropper/app/lib/CropperConfig.scala
+++ b/cropper/app/lib/CropperConfig.scala
@@ -1,5 +1,6 @@
 package lib
 
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
 import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources}
 import com.gu.mediaservice.model.Instance
 
@@ -7,9 +8,10 @@ import java.io.File
 
 
 class CropperConfig(resources: GridConfigResources) extends CommonConfig(resources) {
-  val imgPublishingBucket = string("publishing.image.bucket")
-  val imgPublishingBucketS3Endpoint: String = "s3.amazonaws.com"
-
+  val imgPublishingBucket: S3Bucket = S3Bucket(
+    string("publishing.image.bucket"),
+    S3.AmazonAwsS3Endpoint
+  )
   val canDownloadCrop: Boolean = boolean("canDownloadCrop")
 
   val imgPublishingHost = string("publishing.image.host")
diff --git a/cropper/app/lib/Crops.scala b/cropper/app/lib/Crops.scala
index a58485af65..33ba37ac9b 100644
--- a/cropper/app/lib/Crops.scala
+++ b/cropper/app/lib/Crops.scala
@@ -3,7 +3,7 @@ package lib
 import java.io.File
 import com.gu.mediaservice.lib.metadata.FileMetadataHelper
 import com.gu.mediaservice.lib.Files
-import com.gu.mediaservice.lib.aws.S3
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
 import com.gu.mediaservice.lib.imaging.{ExportResult, ImageOperations}
 import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch}
 import com.gu.mediaservice.model._
@@ -17,7 +17,7 @@ case object InvalidCropRequest extends Exception("Crop request invalid for image
 
 case class MasterCrop(sizing: Future[Asset], file: File, dimensions: Dimensions, aspectRatio: Float)
 
-class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: String)(implicit ec: ExecutionContext) extends GridLogging {
+class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: S3Bucket)(implicit ec: ExecutionContext) extends GridLogging {
   import Files._
 
   private val cropQuality = 75d
diff --git a/cropper/test/lib/CropsTest.scala b/cropper/test/lib/CropsTest.scala
index debab1aab0..8742644e86 100644
--- a/cropper/test/lib/CropsTest.scala
+++ b/cropper/test/lib/CropsTest.scala
@@ -1,5 +1,6 @@
 package lib
 
+import com.gu.mediaservice.lib.aws.S3Bucket
 import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.model._
 import org.scalatest.funspec.AnyFunSpec
@@ -50,7 +51,7 @@ class CropsTest extends AnyFunSpec with Matchers with MockitoSugar {
   private val source: SourceImage = SourceImage("test", mock[Asset], valid = true, mock[ImageMetadata], mock[FileMetadata])
   private val bounds: Bounds = Bounds(10, 20, 30, 40)
   private val outputWidth = 1234
-  private val imageBucket = "crops-bucket"
+  private val imageBucket = S3Bucket("crops-bucket", endpoint = "some-providers-s3-endpoint")
 
   it("should should construct a correct address for a master jpg") {
     val outputFilename = new Crops(config, store, imageOperations, imageBucket)
diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala
index f5d7217d5d..43312a7448 100644
--- a/image-loader/app/lib/ImageLoaderConfig.scala
+++ b/image-loader/app/lib/ImageLoaderConfig.scala
@@ -1,5 +1,7 @@
 package lib
 
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
+
 import java.io.File
 import com.gu.mediaservice.lib.cleanup.{ComposedImageProcessor, ImageProcessor, ImageProcessorResources}
 import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources, ImageProcessorLoader}
@@ -13,8 +15,9 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res
 
   val maybeImageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket")
 
-  val quarantineBucket: Option[String] = stringOpt("s3.quarantine.bucket")
-  val quarantineBucketS3Endpoint: String = "s3.amazonaws.com"
+  val quarantineBucket: Option[S3Bucket] = stringOpt("s3.quarantine.bucket").map { bucket =>
+    S3Bucket(bucket, S3.AmazonAwsS3Endpoint)
+  }
   val uploadToQuarantineEnabled: Boolean = boolean("upload.quarantine.enabled")
 
   val tempDir: File = new File(stringDefault("upload.tmp.dir", "/tmp"))
diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala
index 290b87c589..8b4513e5cc 100644
--- a/image-loader/app/lib/ImageLoaderStore.scala
+++ b/image-loader/app/lib/ImageLoaderStore.scala
@@ -12,7 +12,9 @@ import java.util.Date
 
 class S3FileDoesNotExistException extends Exception()
 
-class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config, imageBucketS3Endpoint = config.imageBucketS3Endpoint, thumbnailBucketS3Endpoint = config.thumbnailBucketS3Endpoint) with GridLogging {
+class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config) with GridLogging {
+
+  private val imageIngestS3Endpoint = AmazonAwsS3Endpoint
 
   private def handleNotFound[T](key: String)(doWork: => T)(loggingIfNotFound: => Unit): T = {
     try {
@@ -35,7 +37,7 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati
 
   def generatePreSignedUploadUrl(filename: String, expiration: ZonedDateTime, uploadedBy: String, mediaId: String)(implicit instance: Instance): String = {
     val request = new GeneratePresignedUrlRequest(
-      config.maybeIngestBucket.get, // bucket
+      config.maybeIngestBucket.get.bucket, // bucket
       s"${instance.id}/$uploadedBy/$filename", // key
     )
       .withMethod(HttpMethod.PUT)
@@ -44,7 +46,7 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati
     // sent by the client in manager.js
     request.putCustomRequestHeader("x-amz-meta-media-id", mediaId)
 
-    generatePresignedRequest(request).toString
+    generatePresignedRequest(request, config.maybeIngestBucket.get).toString
   }
 
   def moveObjectToFailedBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key){
diff --git a/image-loader/app/lib/QuarantineStore.scala b/image-loader/app/lib/QuarantineStore.scala
index e73e28ead2..0c7e5f515f 100644
--- a/image-loader/app/lib/QuarantineStore.scala
+++ b/image-loader/app/lib/QuarantineStore.scala
@@ -3,4 +3,4 @@ package lib.storage
 import lib.ImageLoaderConfig
 import com.gu.mediaservice.lib
 
-class QuarantineStore(config: ImageLoaderConfig) extends lib.ImageQuarantineOperations(config.quarantineBucket.get, config, s3Endpoint = config.quarantineBucketS3Endpoint)
+class QuarantineStore(config: ImageLoaderConfig) extends lib.ImageQuarantineOperations(config.quarantineBucket.get, config)
diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala
index e89c6a19ee..06fe45ad5b 100644
--- a/image-loader/app/model/Projector.scala
+++ b/image-loader/app/model/Projector.scala
@@ -84,7 +84,7 @@ object S3FileExtractedMetadata {
 }
 
 class Projector(config: ImageUploadOpsCfg,
-                s3: AmazonS3,
+                s3: AmazonS3, // TODO Not GCP aware!
                 imageOps: ImageOperations,
                 processor: ImageProcessor,
                 auth: Authentication) extends GridLogging with InstanceForRequest {
@@ -97,11 +97,11 @@ class Projector(config: ImageUploadOpsCfg,
       import ImageIngestOperations.fileKeyFromId
       val s3Key = fileKeyFromId(imageId)
 
-      if (!s3.doesObjectExist(config.originalFileBucket, s3Key))
-        throw new NoSuchImageExistsInS3(config.originalFileBucket, s3Key)
+      if (!s3.doesObjectExist(config.originalFileBucket.bucket, s3Key))
+        throw new NoSuchImageExistsInS3(config.originalFileBucket.bucket, s3Key)
 
       val s3Source = Stopwatch(s"object exists, getting s3 object at s3://${config.originalFileBucket}/$s3Key to perform Image projection"){
-        s3.getObject(config.originalFileBucket, s3Key)
+        s3.getObject(config.originalFileBucket.bucket, s3Key)
       }(logMarker)
 
       try {
@@ -184,18 +184,18 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
   }
 
   private def projectOriginalFileAsS3Model(storableOriginalImage: StorableOriginalImage) =
-    Future.successful(storableOriginalImage.toProjectedS3Object(config.originalFileBucket, config.s3Endpoint))
+    Future.successful(storableOriginalImage.toProjectedS3Object(config.originalFileBucket))
 
   private def projectThumbnailFileAsS3Model(storableThumbImage: StorableThumbImage) =
-    Future.successful(storableThumbImage.toProjectedS3Object(config.thumbBucket, config.s3Endpoint))
+    Future.successful(storableThumbImage.toProjectedS3Object(config.thumbBucket))
 
   private def projectOptimisedPNGFileAsS3Model(storableOptimisedImage: StorableOptimisedImage) =
-    Future.successful(storableOptimisedImage.toProjectedS3Object(config.originalFileBucket, config.s3Endpoint))
+    Future.successful(storableOptimisedImage.toProjectedS3Object(config.originalFileBucket))
 
   private def fetchThumbFile(
     imageId: String, outFile: File, instance: Instance)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
     val key = fileKeyFromId(imageId)(instance)
-    fetchFile(config.thumbBucket, key, outFile)
+    fetchFile(config.thumbBucket.bucket, key, outFile)
   }
 
   private def fetchOptimisedFile(
@@ -203,10 +203,10 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
   )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
     val key = optimisedPngKeyFromId(imageId)(instance)
 
-    fetchFile(config.originalFileBucket, key, outFile)
+    fetchFile(config.originalFileBucket.bucket, key, outFile)
   }
 
-  private def fetchFile(
+  private def fetchFile(  // TODO use S3 trait for GCP support!
     bucket: String, key: String, outFile: File
   )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
     logger.info(logMarker, s"Trying fetch existing image from S3 bucket - $bucket at key $key")
diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala
index e5961dbfee..3bf87d9bf7 100644
--- a/image-loader/app/model/Uploader.scala
+++ b/image-loader/app/model/Uploader.scala
@@ -12,7 +12,7 @@ import com.gu.mediaservice.lib.argo.ArgoHelpers
 import com.gu.mediaservice.lib.auth.Authentication
 import com.gu.mediaservice.lib.auth.Authentication.Principal
 import com.gu.mediaservice.lib.{BrowserViewableImage, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage}
-import com.gu.mediaservice.lib.aws.{S3Object, UpdateMessage}
+import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object, UpdateMessage}
 import com.gu.mediaservice.lib.cleanup.ImageProcessor
 import com.gu.mediaservice.lib.formatting._
 import com.gu.mediaservice.lib.imaging.ImageOperations
@@ -70,9 +70,8 @@ case class ImageUploadOpsCfg(
   thumbWidth: Int,
   thumbQuality: Double,
   transcodedMimeTypes: List[MimeType],
-  originalFileBucket: String,
-  thumbBucket: String,
-  s3Endpoint: String
+  originalFileBucket: S3Bucket,
+  thumbBucket: S3Bucket,
 )
 
 case class ImageUploadOpsDependencies(
@@ -99,7 +98,6 @@ object Uploader extends GridLogging {
       config.transcodedMimeTypes,
       config.imageBucket,
       config.thumbnailBucket,
-      config.thumbnailBucketS3Endpoint
     )
   }
 
diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala
index 9721919ffb..1b76df6239 100644
--- a/image-loader/test/scala/model/ImageUploadTest.scala
+++ b/image-loader/test/scala/model/ImageUploadTest.scala
@@ -5,7 +5,7 @@ import java.net.URI
 import java.util.UUID
 import com.drew.imaging.ImageProcessingException
 import com.gu.mediaservice.lib.{StorableImage, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage}
-import com.gu.mediaservice.lib.aws.{S3Metadata, S3Object, S3ObjectMetadata, S3Ops}
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket, S3Metadata, S3Object, S3ObjectMetadata, S3Ops}
 import com.gu.mediaservice.lib.cleanup.ImageProcessor
 import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.logging.LogMarker
@@ -32,7 +32,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
   private implicit val logMarker: MockLogMarker = new MockLogMarker()
     // For mime type info, see https://github.com/guardian/grid/pull/2568
     val tempDir = new File("/tmp")
-    val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), "img-bucket", "thumb-bucket", "s3.amazonaws.com")
+    val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), S3Bucket("img-bucket", S3.AmazonAwsS3Endpoint), S3Bucket("thumb-bucket", S3.AmazonAwsS3Endpoint))
 
   /**
     * @todo: I flailed about until I found a path that worked, but
@@ -53,7 +53,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
 
     def mockStore = (a: StorableImage) =>
       Future.successful(
-        S3Object("madeupname", "madeupkey", a.file, Some(a.mimeType), None, a.meta, None, "s3.amazonaws.com")
+        S3Object("madeupname", "madeupkey", a.file, Some(a.mimeType), None, a.meta, None, S3.AmazonAwsS3Endpoint)
       )
 
     def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore
diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala
index 1343d92802..e95438c96c 100644
--- a/image-loader/test/scala/model/ProjectorTest.scala
+++ b/image-loader/test/scala/model/ProjectorTest.scala
@@ -7,6 +7,7 @@ import com.amazonaws.services.s3.AmazonS3
 import com.amazonaws.services.s3.model.ObjectMetadata
 import com.gu.mediaservice.GridClient
 import com.gu.mediaservice.lib.auth.Authentication
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
 import com.gu.mediaservice.lib.cleanup.ImageProcessor
 import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap}
@@ -38,7 +39,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc
 
   private val imageOperations = new ImageOperations(ctxPath)
 
-  private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", "thumb-bucket", "s3.amazonaws.com")
+  private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, S3Bucket("img-bucket", S3.AmazonAwsS3Endpoint), S3Bucket("thumb-bucket", S3.AmazonAwsS3Endpoint))
 
   private val s3 = mock[AmazonS3]
   private val auth = mock[Authentication]
diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala
index 4504e466cb..f8812aee3a 100644
--- a/media-api/app/controllers/MediaApi.scala
+++ b/media-api/app/controllers/MediaApi.scala
@@ -9,7 +9,7 @@ import com.gu.mediaservice.lib.auth.Authentication.{Request, _}
 import com.gu.mediaservice.lib.auth.Permissions.{ArchiveImages, DeleteCropsOrUsages, EditMetadata, UploadImages, DeleteImage => DeleteImagePermission}
 import com.gu.mediaservice.lib.auth._
 import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider
-import com.gu.mediaservice.lib.aws.{ContentDisposition, ThrallMessageSender, UpdateMessage}
+import com.gu.mediaservice.lib.aws.{ContentDisposition, S3, ThrallMessageSender, UpdateMessage}
 import com.gu.mediaservice.lib.config.InstanceForRequest
 import com.gu.mediaservice.lib.events.UsageEvents
 import com.gu.mediaservice.lib.formatting.printDateTime
diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala
index 7941496fba..ed8466f138 100644
--- a/media-api/app/lib/MediaApiConfig.scala
+++ b/media-api/app/lib/MediaApiConfig.scala
@@ -1,32 +1,30 @@
 package lib
 
-import com.amazonaws.services.cloudfront.util.SignerUtils
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
 import com.gu.mediaservice.lib.config.{CommonConfigWithElastic, GridConfigResources}
 import com.gu.mediaservice.model.Instance
 import org.joda.time.DateTime
 import scalaz.NonEmptyList
 
-import java.security.PrivateKey
 import scala.util.Try
 
 case class StoreConfig(
-  storeBucket: String,
+  storeBucket: S3Bucket,
   storeKey: String,
-  storeBucketS3Endpoint: String
 )
 
 class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) {
-  val configBucket: String = string("s3.config.bucket")
-  val usageMailBucket: String = string("s3.usagemail.bucket")
+  val configBucket: S3Bucket = S3Bucket(string("s3.config.bucket"), S3.AmazonAwsS3Endpoint)
+  val usageMailBucket: S3Bucket = S3Bucket(string("s3.usagemail.bucket"), S3.AmazonAwsS3Endpoint)
 
   val quotaStoreKey: String = string("quota.store.key")
-  val quotaStoreConfig: StoreConfig = StoreConfig(configBucket, quotaStoreKey, "s3.amazonaws.com")
+  val quotaStoreConfig: StoreConfig = StoreConfig(configBucket, quotaStoreKey)
 
   //Lazy allows this to be empty and not break things unless used somewhere
-  lazy val imgPublishingBucket = string("publishing.image.bucket")
+  lazy val imgPublishingBucket: S3Bucket = S3Bucket(string("publishing.image.bucket"), S3.AmazonAwsS3Endpoint)
 
   val cloudFrontDomainThumbBucket: Option[String]   = stringOpt("cloudfront.domain.thumbbucket")
-  val cloudFrontPrivateKeyBucket: Option[String]    = stringOpt("cloudfront.private-key.bucket")
+  val cloudFrontPrivateKeyBucket: Option[S3Bucket]    = stringOpt("cloudfront.private-key.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint))
   val cloudFrontPrivateKeyBucketKey: Option[String] = stringOpt("cloudfront.private-key.key")
   val cloudFrontKeyPairId: Option[String]           = stringOpt("cloudfront.keypair.id")
 
diff --git a/media-api/app/lib/QuotaStore.scala b/media-api/app/lib/QuotaStore.scala
index ce1e3f8f5c..1606586554 100644
--- a/media-api/app/lib/QuotaStore.scala
+++ b/media-api/app/lib/QuotaStore.scala
@@ -1,16 +1,16 @@
 package lib
 
 import com.gu.mediaservice.lib.BaseStore
+import com.gu.mediaservice.lib.aws.S3Bucket
 import play.api.libs.json.Json
 
 import scala.concurrent.ExecutionContext
 
 class QuotaStore(
                   quotaFile: String,
-                  bucket: String,
+                  bucket: S3Bucket,
                   config: MediaApiConfig,
-                  val s3Endpoint: String,
-                )(implicit ec: ExecutionContext) extends BaseStore[String, SupplierUsageQuota](bucket, config, s3Endpoint)(ec) {
+                )(implicit ec: ExecutionContext) extends BaseStore[String, SupplierUsageQuota](bucket, config)(ec) {
 
   def getQuota: Map[String, SupplierUsageQuota] = store.get()
 
diff --git a/media-api/app/lib/UsageQuota.scala b/media-api/app/lib/UsageQuota.scala
index c508085fc4..0dc049fc77 100644
--- a/media-api/app/lib/UsageQuota.scala
+++ b/media-api/app/lib/UsageQuota.scala
@@ -2,6 +2,7 @@ package lib
 
 import org.apache.pekko.actor.Scheduler
 import com.gu.mediaservice.lib.FeatureToggle
+import com.gu.mediaservice.lib.aws.S3
 import com.gu.mediaservice.model.UsageRights
 
 import scala.concurrent.Await
@@ -17,14 +18,12 @@ class UsageQuota(config: MediaApiConfig, scheduler: Scheduler) {
     config.quotaStoreConfig.storeKey,
     config.quotaStoreConfig.storeBucket,
     config,
-    config.quotaStoreConfig.storeBucketS3Endpoint
   )
 
   val usageStore = new UsageStore(
     config.usageMailBucket,
     config,
     quotaStore,
-    "s3.amazonaws.com"
   )
 
   def scheduleUpdates(): Unit = {
diff --git a/media-api/app/lib/UsageStore.scala b/media-api/app/lib/UsageStore.scala
index a2f3e8130b..c8b55defa0 100644
--- a/media-api/app/lib/UsageStore.scala
+++ b/media-api/app/lib/UsageStore.scala
@@ -1,6 +1,7 @@
 package lib
 
 import com.gu.mediaservice.lib.BaseStore
+import com.gu.mediaservice.lib.aws.S3Bucket
 import com.gu.mediaservice.lib.logging.GridLogging
 import com.gu.mediaservice.model.{Agencies, Agency, UsageRights}
 import org.joda.time.DateTime
@@ -57,11 +58,10 @@ object UsageStore extends GridLogging {
 }
 
 class UsageStore(
-  bucket: String,
+  bucket: S3Bucket,
   config: MediaApiConfig,
   quotaStore: QuotaStore,
-  val s3Endpoint: String
-)(implicit val ec: ExecutionContext) extends BaseStore[String, UsageStatus](bucket, config, s3Endpoint) with GridLogging {
+)(implicit val ec: ExecutionContext) extends BaseStore[String, UsageStatus](bucket, config) with GridLogging {
   import UsageStore._
 
   def getUsageStatusForUsageRights(usageRights: UsageRights): Future[UsageStatus] = {
diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala
index f7dc196512..39fd51ac6b 100644
--- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala
+++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala
@@ -2,6 +2,7 @@ package com.gu.mediaservice.lib.auth.provider
 import com.gu.mediaservice.lib.auth.Authentication.{MachinePrincipal, Principal}
 import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider.{ApiKeyInstance, KindeIdKey}
 import com.gu.mediaservice.lib.auth.{ApiAccessor, KeyStore}
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
 import com.gu.mediaservice.lib.config.InstanceForRequest
 import com.gu.mediaservice.lib.events.UsageEvents
 import com.gu.mediaservice.model.Instance
@@ -25,7 +26,8 @@ class ApiKeyAuthenticationProvider(configuration: Configuration, resources: Auth
   var keyStorePlaceholder: Option[KeyStore] = _
 
   override def initialise(): Unit = {
-    val store = new KeyStore(configuration.get[String]("authKeyStoreBucket"), resources.commonConfig, "s3.amazonaws.com")
+    val authKeyStoreBucket  = S3Bucket(configuration.get[String]("authKeyStoreBucket"), S3.AmazonAwsS3Endpoint)
+    val store = new KeyStore(authKeyStoreBucket, resources.commonConfig)
     store.scheduleUpdates(resources.actorSystem.scheduler)
     keyStorePlaceholder = Some(store)
   }
diff --git a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala
index c5308a584e..220302deee 100644
--- a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala
+++ b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala
@@ -3,6 +3,7 @@ package com.gu.mediaservice.lib.auth
 import org.apache.pekko.actor.ActorSystem
 import com.gu.mediaservice.lib.auth.Authentication.MachinePrincipal
 import com.gu.mediaservice.lib.auth.provider.{ApiKeyAuthenticationProvider, Authenticated, AuthenticationProviderResources, Invalid, NotAuthenticated, NotAuthorised}
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
 import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources}
 import com.gu.mediaservice.lib.events.UsageEvents
 import com.gu.mediaservice.model.Instance
@@ -43,7 +44,7 @@ class ApiKeyAuthenticationProviderTest extends AsyncFreeSpec with Matchers with
       Future.successful(())
     }
 
-    override def keyStore: KeyStore = new KeyStore("not-used", resources.commonConfig, "s3.amazonaws.com") {
+    override def keyStore: KeyStore = new KeyStore(S3Bucket("not-used", S3.AmazonAwsS3Endpoint), resources.commonConfig) {
       override def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = {
         key match {
           case "key-chuckle" => Some(ApiAccessor("brothers", Internal))
diff --git a/thrall/app/controllers/ThrallController.scala b/thrall/app/controllers/ThrallController.scala
index 3ae766bebb..f3881d71ce 100644
--- a/thrall/app/controllers/ThrallController.scala
+++ b/thrall/app/controllers/ThrallController.scala
@@ -7,7 +7,7 @@ import com.amazonaws.services.s3.AmazonS3
 import com.amazonaws.services.s3.model.ListObjectsRequest
 import com.gu.mediaservice.GridClient
 import com.gu.mediaservice.lib.auth.{Authentication, BaseControllerWithLoginRedirects}
-import com.gu.mediaservice.lib.aws.{ThrallMessageSender, UpdateMessage}
+import com.gu.mediaservice.lib.aws.{S3Bucket, ThrallMessageSender, UpdateMessage}
 import com.gu.mediaservice.lib.config.{InstanceForRequest, Services}
 import com.gu.mediaservice.lib.elasticsearch.{NotRunning, Running}
 import com.gu.mediaservice.lib.logging.GridLogging
@@ -41,7 +41,7 @@ class ThrallController(
   override val controllerComponents: ControllerComponents,
   gridClient: GridClient,
   s3: AmazonS3,
-  imageBucket: String,
+  imageBucket: S3Bucket,
 )(implicit val ec: ExecutionContext) extends BaseControllerWithLoginRedirects with GridLogging with InstanceForRequest {
 
   private val numberFormatter: Long => String = java.text.NumberFormat.getIntegerInstance().format
@@ -305,7 +305,7 @@ class ThrallController(
 
     @tailrec
     def getMediaIdsFromS3(all: Seq[String], nextMarker: Option[String])(implicit instance: Instance): Seq[String] = {
-      val baseRequest = new ListObjectsRequest().withBucketName(imageBucket).withPrefix(instance.id + "/")
+      val baseRequest = new ListObjectsRequest().withBucketName(imageBucket.bucket).withPrefix(instance.id + "/")
       val request = nextMarker.map { marker =>
         baseRequest.withMarker(marker)
       }.getOrElse {
diff --git a/thrall/app/lib/ThrallConfig.scala b/thrall/app/lib/ThrallConfig.scala
index 7b4d3b03ae..1a08923f9f 100644
--- a/thrall/app/lib/ThrallConfig.scala
+++ b/thrall/app/lib/ThrallConfig.scala
@@ -1,6 +1,6 @@
 package lib
 
-import com.gu.mediaservice.lib.aws.AwsClientV2BuilderUtils
+import com.gu.mediaservice.lib.aws.{AwsClientV2BuilderUtils, S3, S3Bucket}
 import com.gu.mediaservice.lib.cleanup.ReapableEligibiltyResources
 import com.gu.mediaservice.lib.config.{CommonConfigWithElastic, GridConfigResources, ReapableEligibilityLoader}
 import com.gu.mediaservice.lib.elasticsearch.ReapableEligibility
@@ -56,7 +56,7 @@ object KinesisReceiverConfig {
 }
 
 class ThrallConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) {
-  val maybeReaperBucket: Option[String] = stringOpt("s3.reaper.bucket")
+  val maybeReaperBucket: Option[S3Bucket] = stringOpt("s3.reaper.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint))
   val maybeReaperCountPerRun: Option[Int] = intOpt("reaper.countPerRun")
 
   val metadataTopicArn: String = string("indexed.image.sns.topic.arn")
diff --git a/thrall/app/lib/ThrallStore.scala b/thrall/app/lib/ThrallStore.scala
index 4deb3e5444..a221c0471c 100644
--- a/thrall/app/lib/ThrallStore.scala
+++ b/thrall/app/lib/ThrallStore.scala
@@ -2,4 +2,4 @@ package lib
 
 import com.gu.mediaservice.lib
 
-class ThrallStore(config: ThrallConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config, config.isVersionedS3, config.imageBucketS3Endpoint, config.thumbnailBucketS3Endpoint)
+class ThrallStore(config: ThrallConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config, config.isVersionedS3)

From 61bf0ecfeb718bb0bdd561c787447d317de8064b Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 1 Dec 2024 13:18:13 +0000
Subject: [PATCH 05/45] Reindex is bucket aware (what was it doing before?)

---
 .../main/scala/com/gu/mediaservice/lib/aws/S3.scala |  4 ++++
 thrall/app/ThrallComponents.scala                   |  4 +++-
 thrall/app/controllers/ThrallController.scala       | 13 +++++++------
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index ea1c6ba2f6..ffb7959c15 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -163,6 +163,10 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     clientFor(bucket).listObjects(bucket.bucket, prefix)
   }
 
+  def listObjects(bucket: S3Bucket, request: ListObjectsRequest): ObjectListing = {
+    clientFor(bucket).listObjects(request)
+  }
+
   def listObjectKeys(bucket: S3Bucket): Seq[String] = {
     clientFor(bucket).listObjects(bucket.bucket).getObjectSummaries.asScala.map(_.getKey).toSeq
   }
diff --git a/thrall/app/ThrallComponents.scala b/thrall/app/ThrallComponents.scala
index 7bf5eace79..426b7b5f5d 100644
--- a/thrall/app/ThrallComponents.scala
+++ b/thrall/app/ThrallComponents.scala
@@ -2,6 +2,7 @@ import org.apache.pekko.Done
 import org.apache.pekko.stream.scaladsl.Source
 import com.gu.kinesis.{KinesisRecord, KinesisSource, ConsumerConfig => KclPekkoStreamConfig}
 import com.gu.mediaservice.GridClient
+import com.gu.mediaservice.lib.aws.ThrallMessageSender
 import com.gu.mediaservice.lib.aws.{S3Ops, ThrallMessageSender}
 import com.gu.mediaservice.lib.instances.{Instances, InstancesClient}
 import com.gu.mediaservice.lib.logging.MarkerMap
@@ -22,6 +23,7 @@ import software.amazon.awssdk.services.sqs.model.GetQueueUrlRequest
 import scala.concurrent.{Await, Future}
 import scala.concurrent.duration._
 import scala.language.postfixOps
+import com.gu.mediaservice.lib.aws.S3
 
 class ThrallComponents(context: Context) extends GridComponents(context, new ThrallConfig(_)) with StrictLogging with AssetsComponents
   with Instances {
@@ -91,7 +93,7 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr
 
   val streamRunning: Future[Done] = thrallStreamProcessor.run()
 
-  val s3 = S3Ops.buildS3Client(config)
+  val s3 = new S3(config)
 
   Source.repeat(()).throttle(1, per = 5.minute).map(_ => {
     implicit val logMarker: MarkerMap = MarkerMap()
diff --git a/thrall/app/controllers/ThrallController.scala b/thrall/app/controllers/ThrallController.scala
index f3881d71ce..e354416946 100644
--- a/thrall/app/controllers/ThrallController.scala
+++ b/thrall/app/controllers/ThrallController.scala
@@ -3,11 +3,10 @@ package controllers
 import org.apache.pekko.actor.ActorSystem
 import org.apache.pekko.stream.Materializer
 import org.apache.pekko.stream.scaladsl.{Sink, Source}
-import com.amazonaws.services.s3.AmazonS3
 import com.amazonaws.services.s3.model.ListObjectsRequest
 import com.gu.mediaservice.GridClient
 import com.gu.mediaservice.lib.auth.{Authentication, BaseControllerWithLoginRedirects}
-import com.gu.mediaservice.lib.aws.{S3Bucket, ThrallMessageSender, UpdateMessage}
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket, ThrallMessageSender, UpdateMessage}
 import com.gu.mediaservice.lib.config.{InstanceForRequest, Services}
 import com.gu.mediaservice.lib.elasticsearch.{NotRunning, Running}
 import com.gu.mediaservice.lib.logging.GridLogging
@@ -40,7 +39,7 @@ class ThrallController(
   override val services: Services,
   override val controllerComponents: ControllerComponents,
   gridClient: GridClient,
-  s3: AmazonS3,
+  s3: S3,
   imageBucket: S3Bucket,
 )(implicit val ec: ExecutionContext) extends BaseControllerWithLoginRedirects with GridLogging with InstanceForRequest {
 
@@ -150,7 +149,7 @@ class ThrallController(
   def startMigration = withLoginRedirectAsync { implicit request =>
     val instance = instanceOf(request)
 
-    if(Form(single("start-confirmation" -> text)).bindFromRequest().get != "start"){
+    if (Form(single("start-confirmation" -> text)).bindFromRequest().get != "start") {
       Future.successful(BadRequest("you did not enter 'start' in the text box"))
     } else {
       val msgFailedToFetchIndex = s"Could not fetch ES index details for alias '${es.imagesMigrationAlias(instance)}'"
@@ -192,7 +191,7 @@ class ThrallController(
 
   def completeMigration(): Action[AnyContent] = withLoginRedirectAsync { implicit request =>
     val instance = instanceOf(request)
-    if(Form(single("complete-confirmation" -> text)).bindFromRequest().get != "complete"){
+    if (Form(single("complete-confirmation" -> text)).bindFromRequest().get != "complete") {
       Future.successful(BadRequest("you did not enter 'complete' in the text box"))
     } else {
       es.refreshAndRetrieveMigrationStatus(instance) match {
@@ -312,7 +311,7 @@ class ThrallController(
         baseRequest
       }
 
-      val listing = s3.listObjects(request)
+      val listing = s3.listObjects(imageBucket, request)
       val keys = listing.getObjectSummaries.asScala.flatMap { s3Object =>
         logger.info("Reindexing s3 key: " + s3Object.getKey)
         s3Object.getKey.split("/").lastOption
@@ -325,7 +324,9 @@ class ThrallController(
       }
     }
 
+    logger.info(s"Reindex requested for instance ${instance.id}")
     val mediaIds = getMediaIdsFromS3(Seq.empty, None)
+    logger.info(s"Reindexing ${mediaIds.size} images for instance ${instance.id}")
     mediaIds.foreach { mediaId =>
       Await.result(reindexImage(mediaId), Duration(10, TimeUnit.SECONDS))
     }

From c49ad22d1148b00f9d81abd8d6bf6c2493c57f24 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 28 Mar 2025 21:06:34 +0000
Subject: [PATCH 06/45] Image projector is bucket S3 endpoint aware.

---
 image-loader/app/ImageLoaderComponents.scala  |  5 ++--
 image-loader/app/model/Projector.scala        | 27 +++++++++----------
 .../test/scala/model/ProjectorTest.scala      |  3 +--
 3 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala
index 4c26ac9eb3..c04d624203 100644
--- a/image-loader/app/ImageLoaderComponents.scala
+++ b/image-loader/app/ImageLoaderComponents.scala
@@ -1,5 +1,5 @@
 import com.gu.mediaservice.GridClient
-import com.gu.mediaservice.lib.aws.SimpleSqsMessageConsumer
+import com.gu.mediaservice.lib.aws.{S3, SimpleSqsMessageConsumer}
 import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.logging.GridLogging
 import com.gu.mediaservice.lib.play.GridComponents
@@ -26,7 +26,8 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne
   val notifications = new Notifications(config)
   val downloader = new Downloader()(ec,wsClient)
   val uploader = new Uploader(store, config, imageOperations, notifications, imageProcessor)
-  val projector = Projector(config, imageOperations, imageProcessor, auth)
+  val s3 = new S3(config)
+  val projector = Projector(config, imageOperations, imageProcessor, auth, s3)
   val quarantineUploader: Option[QuarantineUploader] = (config.uploadToQuarantineEnabled, config.quarantineBucket) match {
     case (true, Some(bucketName)) =>{
       val quarantineStore = new QuarantineStore(config)
diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala
index 06fe45ad5b..0fb8f8c0dc 100644
--- a/image-loader/app/model/Projector.scala
+++ b/image-loader/app/model/Projector.scala
@@ -1,13 +1,12 @@
 package model
 
 import java.io.{File, FileOutputStream}
-import com.amazonaws.services.s3.AmazonS3
 import com.gu.mediaservice.{GridClient, ImageDataMerger}
 import com.gu.mediaservice.lib.auth.Authentication
-import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, S3Object => AwsS3Object}
+import com.amazonaws.services.s3.model.{ObjectMetadata, S3Object => AwsS3Object}
 import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId}
 import com.gu.mediaservice.lib.{ImageIngestOperations, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage}
-import com.gu.mediaservice.lib.aws.S3Ops
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
 import com.gu.mediaservice.lib.cleanup.ImageProcessor
 import com.gu.mediaservice.lib.config.InstanceForRequest
 import com.gu.mediaservice.lib.imaging.ImageOperations
@@ -30,8 +29,8 @@ object Projector {
 
   import Uploader.toImageUploadOpsCfg
 
-  def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication)(implicit ec: ExecutionContext): Projector
-  = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth)
+  def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, s3: S3)(implicit ec: ExecutionContext): Projector
+  = new Projector(toImageUploadOpsCfg(config), s3, imageOps, processor, auth)
 }
 
 case class S3FileExtractedMetadata(
@@ -84,7 +83,7 @@ object S3FileExtractedMetadata {
 }
 
 class Projector(config: ImageUploadOpsCfg,
-                s3: AmazonS3, // TODO Not GCP aware!
+                s3: S3,
                 imageOps: ImageOperations,
                 processor: ImageProcessor,
                 auth: Authentication) extends GridLogging with InstanceForRequest {
@@ -97,11 +96,11 @@ class Projector(config: ImageUploadOpsCfg,
       import ImageIngestOperations.fileKeyFromId
       val s3Key = fileKeyFromId(imageId)
 
-      if (!s3.doesObjectExist(config.originalFileBucket.bucket, s3Key))
+      if (!s3.doesObjectExist(config.originalFileBucket, s3Key))
         throw new NoSuchImageExistsInS3(config.originalFileBucket.bucket, s3Key)
 
       val s3Source = Stopwatch(s"object exists, getting s3 object at s3://${config.originalFileBucket}/$s3Key to perform Image projection"){
-        s3.getObject(config.originalFileBucket.bucket, s3Key)
+        s3.getObject(config.originalFileBucket, s3Key)
       }(logMarker)
 
       try {
@@ -162,7 +161,7 @@ class Projector(config: ImageUploadOpsCfg,
 class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
                                imageOps: ImageOperations,
                                processor: ImageProcessor,
-                               s3: AmazonS3
+                               s3: S3
 ) extends GridLogging {
 
   import Uploader.{fromUploadRequestShared, toMetaMap}
@@ -195,7 +194,7 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
   private def fetchThumbFile(
     imageId: String, outFile: File, instance: Instance)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
     val key = fileKeyFromId(imageId)(instance)
-    fetchFile(config.thumbBucket.bucket, key, outFile)
+    fetchFile(config.thumbBucket, key, outFile)
   }
 
   private def fetchOptimisedFile(
@@ -203,11 +202,11 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
   )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
     val key = optimisedPngKeyFromId(imageId)(instance)
 
-    fetchFile(config.originalFileBucket.bucket, key, outFile)
+    fetchFile(config.originalFileBucket, key, outFile)
   }
 
-  private def fetchFile(  // TODO use S3 trait for GCP support!
-    bucket: String, key: String, outFile: File
+  private def fetchFile(
+                         bucket: S3Bucket, key: String, outFile: File
   )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
     logger.info(logMarker, s"Trying fetch existing image from S3 bucket - $bucket at key $key")
     val doesFileExist = Future { s3.doesObjectExist(bucket, key) } recover { case _ => false }
@@ -216,7 +215,7 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
         logger.warn(logMarker, s"image did not exist in bucket $bucket at key $key")
         Future.successful(None) // falls back to creating from original file
       case true =>
-        val obj = s3.getObject(new GetObjectRequest(bucket, key))
+        val obj = s3.getObject(bucket, key)
         val fos = new FileOutputStream(outFile)
         try {
           IOUtils.copy(obj.getObjectContent, fos)
diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala
index e95438c96c..073949a042 100644
--- a/image-loader/test/scala/model/ProjectorTest.scala
+++ b/image-loader/test/scala/model/ProjectorTest.scala
@@ -3,7 +3,6 @@ package model
 import java.io.File
 import java.net.URI
 import java.util.{Date, UUID}
-import com.amazonaws.services.s3.AmazonS3
 import com.amazonaws.services.s3.model.ObjectMetadata
 import com.gu.mediaservice.GridClient
 import com.gu.mediaservice.lib.auth.Authentication
@@ -41,7 +40,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc
 
   private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, S3Bucket("img-bucket", S3.AmazonAwsS3Endpoint), S3Bucket("thumb-bucket", S3.AmazonAwsS3Endpoint))
 
-  private val s3 = mock[AmazonS3]
+  private val s3 = mock[S3]
   private val auth = mock[Authentication]
   private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth)
 

From 594bd65ddb313bc17a90b2f149c0dcaeec2b7465 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 29 Nov 2024 22:17:03 +0000
Subject: [PATCH 07/45] Image bucket takes end point config from config.

Update fixture config with new bucket conf.
---
 .../scala/com/gu/mediaservice/lib/config/CommonConfig.scala   | 2 +-
 common-lib/src/test/resources/application.conf                | 4 +++-
 media-api/test/lib/elasticsearch/Fixtures.scala               | 3 ++-
 rest-lib/src/test/resources/application.conf                  | 3 +++
 4 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
index 8745bc0710..3833fea984 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
@@ -69,7 +69,7 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
 
   val services = new SingleHostServices(domainRoot)
 
-  val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket"), S3.AmazonAwsS3Endpoint)
+  val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket.name") ,stringOpt("s3.image.bucket.endpoint").get)
   val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket"), S3.AmazonAwsS3Endpoint)
   val thumbnailBucketS3Endpoint: String = S3.AmazonAwsS3Endpoint
 
diff --git a/common-lib/src/test/resources/application.conf b/common-lib/src/test/resources/application.conf
index e3724393da..c178da2149 100644
--- a/common-lib/src/test/resources/application.conf
+++ b/common-lib/src/test/resources/application.conf
@@ -62,6 +62,8 @@ instance.service.instances=""
 
 usageEvents.queue.name=""
 
-s3.image.bucket="images"
+s3.image.bucket.name="images"
+s3.image.bucket.endpoint="some-providers-s3-endpoint"
 
 s3.thumb.bucket="thumbs"
+
diff --git a/media-api/test/lib/elasticsearch/Fixtures.scala b/media-api/test/lib/elasticsearch/Fixtures.scala
index 84d2fa8eee..16f558c228 100644
--- a/media-api/test/lib/elasticsearch/Fixtures.scala
+++ b/media-api/test/lib/elasticsearch/Fixtures.scala
@@ -35,7 +35,8 @@ trait Fixtures {
     "es.index.aliases.current",
     "es.index.aliases.migration",
     "es6.url",
-    "s3.image.bucket",
+    "s3.image.bucket.name",
+    "s3.image.bucket.endpoint",
     "s3.thumb.bucket",
     "grid.stage",
     "grid.appName",
diff --git a/rest-lib/src/test/resources/application.conf b/rest-lib/src/test/resources/application.conf
index 930c6a00d5..cfd0579a07 100644
--- a/rest-lib/src/test/resources/application.conf
+++ b/rest-lib/src/test/resources/application.conf
@@ -3,3 +3,6 @@ grid.appName: "test"
 thrall.kinesis.stream.name: "not-used"
 thrall.kinesis.lowPriorityStream.name: "not-used"
 domain.root: "notused.example.com"
+s3.image.bucket.name: images
+s3.image.bucket.endpoint: some-providers-s3-endpoint
+s3.thumb.bucket: thumbs

From 46d70834aed01eddb1c592853b1ff31bd34d1c47 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 17 Jan 2025 18:12:11 +0000
Subject: [PATCH 08/45] Ingest and rejected buckets have configurable end
 points.

---
 .../mediaservice/lib/config/CommonConfig.scala   | 16 +++++++++++++---
 image-loader/app/lib/ImageLoaderStore.scala      |  2 --
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
index 3833fea984..46fb530df0 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
@@ -53,8 +53,18 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
   lazy val softDeletedMetadataTable: String = string("dynamo.table.softDelete.metadata")
 
   val maybeIngestSqsQueueUrl: Option[String] = stringOpt("sqs.ingest.queue.url")
-  val maybeIngestBucket: Option[S3Bucket] = stringOpt("s3.ingest.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint))
-  val maybeFailBucket: Option[S3Bucket] = stringOpt("s3.fail.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint))
+  val maybeIngestBucket: Option[S3Bucket] = for {
+    ingestBucket <- stringOpt("s3.ingest.bucket.name")
+    ingestBucketEndpoint <- stringOpt("s3.ingest.bucket.endpoint")
+  } yield {
+    S3Bucket(ingestBucket, ingestBucketEndpoint)
+  }
+  val maybeFailBucket: Option[S3Bucket] = for {
+    failBucket <- stringOpt("s3.fail.bucket.name")
+    failBucketEndpoint <- stringOpt("s3.fail.bucket.endpoint")
+  } yield {
+    S3Bucket(failBucket, failBucketEndpoint)
+  }
 
   val maybeUploadLimitInBytes: Option[Int] = intOpt("upload.limit.mb").map(_ * 1024 * 1024)
 
@@ -69,7 +79,7 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
 
   val services = new SingleHostServices(domainRoot)
 
-  val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket.name") ,stringOpt("s3.image.bucket.endpoint").get)
+  val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket.name"), string("s3.image.bucket.endpoint"))
   val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket"), S3.AmazonAwsS3Endpoint)
   val thumbnailBucketS3Endpoint: String = S3.AmazonAwsS3Endpoint
 
diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala
index 8b4513e5cc..83ed57da63 100644
--- a/image-loader/app/lib/ImageLoaderStore.scala
+++ b/image-loader/app/lib/ImageLoaderStore.scala
@@ -14,8 +14,6 @@ class S3FileDoesNotExistException extends Exception()
 
 class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config) with GridLogging {
 
-  private val imageIngestS3Endpoint = AmazonAwsS3Endpoint
-
   private def handleNotFound[T](key: String)(doWork: => T)(loggingIfNotFound: => Unit): T = {
     try {
       doWork

From 1fbeb48396cbcccfb44a2a21defa3f1ecd476097 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 24 Jan 2025 17:27:09 +0000
Subject: [PATCH 09/45] Thumbs bucket has configurable end point.

---
 .../scala/com/gu/mediaservice/lib/config/CommonConfig.scala    | 3 +--
 common-lib/src/test/resources/application.conf                 | 3 ++-
 media-api/test/lib/elasticsearch/Fixtures.scala                | 3 ++-
 rest-lib/src/test/resources/application.conf                   | 3 ++-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
index 46fb530df0..06b6700bfc 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
@@ -80,8 +80,7 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
   val services = new SingleHostServices(domainRoot)
 
   val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket.name"), string("s3.image.bucket.endpoint"))
-  val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket"), S3.AmazonAwsS3Endpoint)
-  val thumbnailBucketS3Endpoint: String = S3.AmazonAwsS3Endpoint
+  val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket.name"), string("s3.thumb.bucket.endpoint"))
 
   /**
    * Load in a list of domain metadata specifications from configuration. For example:
diff --git a/common-lib/src/test/resources/application.conf b/common-lib/src/test/resources/application.conf
index c178da2149..b5c3e1aee2 100644
--- a/common-lib/src/test/resources/application.conf
+++ b/common-lib/src/test/resources/application.conf
@@ -65,5 +65,6 @@ usageEvents.queue.name=""
 s3.image.bucket.name="images"
 s3.image.bucket.endpoint="some-providers-s3-endpoint"
 
-s3.thumb.bucket="thumbs"
+s3.thumb.bucket.name="thumbs"
+s3.thumb.bucket.endpoint="some-providers-s3-endpoint"
 
diff --git a/media-api/test/lib/elasticsearch/Fixtures.scala b/media-api/test/lib/elasticsearch/Fixtures.scala
index 16f558c228..df7334f978 100644
--- a/media-api/test/lib/elasticsearch/Fixtures.scala
+++ b/media-api/test/lib/elasticsearch/Fixtures.scala
@@ -37,7 +37,8 @@ trait Fixtures {
     "es6.url",
     "s3.image.bucket.name",
     "s3.image.bucket.endpoint",
-    "s3.thumb.bucket",
+    "s3.thumb.bucket.name",
+    "s3.thumb.bucket.endpoint",
     "grid.stage",
     "grid.appName",
     "instance.service.my",
diff --git a/rest-lib/src/test/resources/application.conf b/rest-lib/src/test/resources/application.conf
index cfd0579a07..7e55f72bb2 100644
--- a/rest-lib/src/test/resources/application.conf
+++ b/rest-lib/src/test/resources/application.conf
@@ -5,4 +5,5 @@ thrall.kinesis.lowPriorityStream.name: "not-used"
 domain.root: "notused.example.com"
 s3.image.bucket.name: images
 s3.image.bucket.endpoint: some-providers-s3-endpoint
-s3.thumb.bucket: thumbs
+s3.thumb.bucket.name: thumbs
+s3.thumb.bucket.endpoint: some-providers-s3-endpoint

From 20a5e86bc340f788df8be5300e8ba4448726e228 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 29 Nov 2024 21:12:03 +0000
Subject: [PATCH 10/45] Configure a GCP S3 client and selectively use it based
 on the bucket endpoint.

Not great.

Crops require mock S3 config. Presence of Google details on this AWS path not great. Suggests client might be a component of bucket.
---
 .../com/gu/mediaservice/lib/aws/S3.scala      | 35 +++++++++++++++++--
 .../lib/config/CommonConfig.scala             |  3 ++
 cropper/test/lib/CropsTest.scala              | 13 +++++--
 3 files changed, 45 insertions(+), 6 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index ffb7959c15..f94ddb646c 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -1,5 +1,7 @@
 package com.gu.mediaservice.lib.aws
 
+import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
 import com.amazonaws.services.s3.model._
 import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder, model}
 import com.amazonaws.util.IOUtils
@@ -67,13 +69,21 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
 
   val AmazonAwsS3Endpoint: String = S3.AmazonAwsS3Endpoint
 
-  private lazy val amazonS3: AmazonS3 = S3Ops.buildS3Client(config)
+  private val amazonS3: AmazonS3 = S3Ops.buildS3Client(config)
+  private val googleS3: Option[AmazonS3] = S3Ops.buildGoogleS3Client(config)
+
   // also create a legacy client that uses v2 signatures for URL signing
   private lazy val legacySigningClient: AmazonS3 = S3Ops.buildS3Client(config, forceV2Sigs = true)
 
   private def clientFor(bucket: S3Bucket): AmazonS3 = {
-    logger.info("Client for: " + bucket.endpoint)
-    amazonS3
+    (bucket.endpoint match {
+      case "storage.googleapis.com" =>
+        googleS3
+      case _ =>
+        Some(amazonS3)
+    }).getOrElse {
+      amazonS3
+    }
   }
 
   def signUrl(bucket: S3Bucket, url: URI, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = {
@@ -246,6 +256,25 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
 }
 
 object S3Ops {
+  def buildGoogleS3Client(config: CommonConfig): Option[AmazonS3] = {
+    config.googleS3AccessKey.flatMap { accessKey =>
+      config.googleS3SecretKey.map { secretKey =>
+        val endpointConfig = new EndpointConfiguration("https://storage.googleapis.com", null)
+        // create credentials provider
+        val credentials = new BasicAWSCredentials(accessKey, secretKey)
+        val credentialsProvider = new AWSStaticCredentialsProvider(credentials)
+        // create a client config
+        val clientConfig = new ClientConfiguration()
+
+        val clientBuilder = AmazonS3ClientBuilder.standard()
+        clientBuilder.setEndpointConfiguration(endpointConfig)
+        clientBuilder.withCredentials(credentialsProvider)
+        clientBuilder.withClientConfiguration(clientConfig)
+        clientBuilder.build()
+      }
+    }
+  }
+
   def buildS3Client(config: CommonConfig, forceV2Sigs: Boolean = false, localstackAware: Boolean = true, maybeRegionOverride: Option[String] = None): AmazonS3 = {
 
     val clientConfig = new ClientConfiguration()
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
index 06b6700bfc..3b0b448cbd 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
@@ -82,6 +82,9 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
   val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket.name"), string("s3.image.bucket.endpoint"))
   val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket.name"), string("s3.thumb.bucket.endpoint"))
 
+  val googleS3AccessKey = stringOpt("s3.accessKey")
+  val googleS3SecretKey = stringOpt("s3.secretKey")
+
   /**
    * Load in a list of domain metadata specifications from configuration. For example:
    * {{{
diff --git a/cropper/test/lib/CropsTest.scala b/cropper/test/lib/CropsTest.scala
index 8742644e86..1a15ba9f52 100644
--- a/cropper/test/lib/CropsTest.scala
+++ b/cropper/test/lib/CropsTest.scala
@@ -1,8 +1,9 @@
 package lib
 
-import com.gu.mediaservice.lib.aws.S3Bucket
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
 import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.model._
+import org.mockito.Mockito.when
 import org.scalatest.funspec.AnyFunSpec
 import org.scalatest.matchers.should.Matchers
 import org.scalatestplus.mockito.MockitoSugar
@@ -45,13 +46,19 @@ class CropsTest extends AnyFunSpec with Matchers with MockitoSugar {
     Crops.cropType(Tiff, "TrueColor", hasAlpha = false) shouldBe Jpeg
   }
 
-  private val config = mock[CropperConfig]
+  private val config = {
+    val mockConfig = mock[CropperConfig]
+    when(mockConfig.awsRegion).thenReturn("eu-west-1")
+    when(mockConfig.googleS3AccessKey).thenReturn(None)
+    when(mockConfig.googleS3SecretKey).thenReturn(None)
+    mockConfig
+  }
   private val store = mock[CropStore]
   private val imageOperations: ImageOperations = mock[ImageOperations]
   private val source: SourceImage = SourceImage("test", mock[Asset], valid = true, mock[ImageMetadata], mock[FileMetadata])
   private val bounds: Bounds = Bounds(10, 20, 30, 40)
   private val outputWidth = 1234
-  private val imageBucket = S3Bucket("crops-bucket", endpoint = "some-providers-s3-endpoint")
+  private val imageBucket = S3Bucket("crops-bucket", S3.AmazonAwsS3Endpoint)
 
   it("should should construct a correct address for a master jpg") {
     val outputFilename = new Crops(config, store, imageOperations, imageBucket)

From ba7a9af7d2e1dc0515f5108b73f5d0725a0ca510 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sat, 30 Nov 2024 10:00:19 +0000
Subject: [PATCH 11/45] v2 sigs on all AWS buckets for better caching.

---
 common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index f94ddb646c..fd6e72a1ed 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -69,7 +69,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
 
   val AmazonAwsS3Endpoint: String = S3.AmazonAwsS3Endpoint
 
-  private val amazonS3: AmazonS3 = S3Ops.buildS3Client(config)
+  private val amazonS3: AmazonS3 = S3Ops.buildS3Client(config, forceV2Sigs = true)
   private val googleS3: Option[AmazonS3] = S3Ops.buildGoogleS3Client(config)
 
   // also create a legacy client that uses v2 signatures for URL signing

From a8f0a7dfbc9dd4ee66384fac14b4fd53b8f842d8 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 15 Dec 2024 20:10:47 +0000
Subject: [PATCH 12/45] Signed S3 URLs come from bucket's client.

---
 .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala    | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index fd6e72a1ed..384b5d84bb 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -72,9 +72,6 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
   private val amazonS3: AmazonS3 = S3Ops.buildS3Client(config, forceV2Sigs = true)
   private val googleS3: Option[AmazonS3] = S3Ops.buildGoogleS3Client(config)
 
-  // also create a legacy client that uses v2 signatures for URL signing
-  private lazy val legacySigningClient: AmazonS3 = S3Ops.buildS3Client(config, forceV2Sigs = true)
-
   private def clientFor(bucket: S3Bucket): AmazonS3 = {
     (bucket.endpoint match {
       case "storage.googleapis.com" =>
@@ -95,7 +92,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     val headers = new ResponseHeaderOverrides().withContentDisposition(contentDisposition)
 
     val request = new GeneratePresignedUrlRequest(bucket.bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers)
-    legacySigningClient.generatePresignedUrl(request).toExternalForm
+    clientFor(bucket).generatePresignedUrl(request).toExternalForm
   }
 
   def signUrlTony(bucket: S3Bucket, url: URI, expiration: DateTime = cachableExpiration()): URL = {
@@ -103,7 +100,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     val key: Key = url.getPath.drop(1)
 
     val request = new GeneratePresignedUrlRequest(bucket.bucket, key).withExpiration(expiration.toDate)
-    legacySigningClient.generatePresignedUrl(request)
+    clientFor(bucket).generatePresignedUrl(request)
   }
 
   def copyObject(sourceBucket: S3Bucket, destinationBucket: S3Bucket, key: String): CopyObjectResult = {

From b4faf720030f58c5560b5e2beaa78792bfd7eb1e Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Thu, 23 Jan 2025 22:50:29 +0000
Subject: [PATCH 13/45] Setup a local Minio S3 client.

Use single host URLs for local S3.
---
 .../com/gu/mediaservice/lib/aws/S3.scala      | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index 384b5d84bb..b450113bb7 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -71,11 +71,14 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
 
   private val amazonS3: AmazonS3 = S3Ops.buildS3Client(config, forceV2Sigs = true)
   private val googleS3: Option[AmazonS3] = S3Ops.buildGoogleS3Client(config)
+  private val localS3: Option[AmazonS3] = S3Ops.buildLocalS3Client(config)
 
   private def clientFor(bucket: S3Bucket): AmazonS3 = {
     (bucket.endpoint match {
       case "storage.googleapis.com" =>
         googleS3
+      case "10.0.45.121:32090" =>
+        localS3
       case _ =>
         Some(amazonS3)
     }).getOrElse {
@@ -272,6 +275,26 @@ object S3Ops {
     }
   }
 
+  def buildLocalS3Client(config: CommonConfig): Option[AmazonS3] = {
+    config.googleS3AccessKey.flatMap { accessKey =>
+      config.googleS3SecretKey.map { secretKey =>
+        val endpointConfig = new EndpointConfiguration("http://10.0.45.121:32090", null)
+        // create credentials provider
+        val credentials = new BasicAWSCredentials(accessKey, secretKey)
+        val credentialsProvider = new AWSStaticCredentialsProvider(credentials)
+        // create a client config
+        val clientConfig = new ClientConfiguration()
+
+        val clientBuilder = AmazonS3ClientBuilder.standard()
+        clientBuilder.setEndpointConfiguration(endpointConfig)
+        clientBuilder.withCredentials(credentialsProvider)
+        clientBuilder.withClientConfiguration(clientConfig)
+        clientBuilder.withPathStyleAccessEnabled(true)
+        clientBuilder.build()
+      }
+    }
+  }
+
   def buildS3Client(config: CommonConfig, forceV2Sigs: Boolean = false, localstackAware: Boolean = true, maybeRegionOverride: Option[String] = None): AmazonS3 = {
 
     val clientConfig = new ClientConfiguration()

From 39a6c92a3f514437164d43b42696adbb553827b8 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Thu, 23 Jan 2025 23:34:54 +0000
Subject: [PATCH 14/45] Local S3.

---
 .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala     | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index b450113bb7..efe78e5577 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -20,7 +20,11 @@ case class S3Object(uri: URI, size: Long, metadata: S3Metadata)
 
 object S3Object {
   private def objectUrl(bucket: String, key: String, s3Endpoint: String): URI = {
-    val bucketUrl = s"$bucket.$s3Endpoint"
+    val bucketUrl = if (s3Endpoint == "10.0.45.121:32090") {
+      s"$s3Endpoint/$bucket"
+    } else {
+      s"$bucket.$s3Endpoint"
+    }
     new URI("http", bucketUrl, s"/$key", null)
   }
 

From 0ad4670688e7035651c544d881ddfeb8857ccede Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Thu, 23 Jan 2025 23:49:29 +0000
Subject: [PATCH 15/45] Local S3.

---
 .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala    | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index efe78e5577..07ab742caf 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -21,11 +21,12 @@ case class S3Object(uri: URI, size: Long, metadata: S3Metadata)
 object S3Object {
   private def objectUrl(bucket: String, key: String, s3Endpoint: String): URI = {
     val bucketUrl = if (s3Endpoint == "10.0.45.121:32090") {
-      s"$s3Endpoint/$bucket"
+      new URI(s"http://$s3Endpoint/$bucket/$key")
     } else {
-      s"$bucket.$s3Endpoint"
+      val bucketHost = s"$bucket.$s3Endpoint"
+      new URI("http", bucketHost, s"/$key", null)
     }
-    new URI("http", bucketUrl, s"/$key", null)
+    bucketUrl
   }
 
   def apply(bucket: String, key: String, size: Long, metadata: S3Metadata, s3Endpoint: String): S3Object =

From 1c1549e5505e6212ca618a98272c7a7ce816617c Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 24 Jan 2025 08:51:33 +0000
Subject: [PATCH 16/45] =?UTF-8?q?Local=20S=C2=A3=20urls=20to=20keys.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../com/gu/mediaservice/lib/aws/S3.scala      | 19 ++++---------------
 .../mediaservice/lib/aws/S3KeyFromURL.scala   | 19 +++++++++++++++++++
 cropper/app/lib/CropStore.scala               | 11 +++++++----
 cropper/app/lib/Crops.scala                   |  8 +++++---
 media-api/app/controllers/MediaApi.scala      | 14 +++++++++-----
 media-api/app/lib/ImageResponse.scala         | 10 ++++++----
 6 files changed, 50 insertions(+), 31 deletions(-)
 create mode 100644 common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index 07ab742caf..b6df0cfe70 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -91,10 +91,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     }
   }
 
-  def signUrl(bucket: S3Bucket, url: URI, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = {
-    // get path and remove leading `/`
-    val key: Key = url.getPath.drop(1)
-
+  def signUrl(bucket: S3Bucket, key: String, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = {
     val contentDisposition = getContentDisposition(image, imageType, config.shortenDownloadFilename)
 
     val headers = new ResponseHeaderOverrides().withContentDisposition(contentDisposition)
@@ -103,10 +100,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     clientFor(bucket).generatePresignedUrl(request).toExternalForm
   }
 
-  def signUrlTony(bucket: S3Bucket, url: URI, expiration: DateTime = cachableExpiration()): URL = {
-    // get path and remove leading `/`
-    val key: Key = url.getPath.drop(1)
-
+  def signUrlTony(bucket: S3Bucket, key: String, expiration: DateTime = cachableExpiration()): URL = {
     val request = new GeneratePresignedUrlRequest(bucket.bucket, key).withExpiration(expiration.toDate)
     clientFor(bucket).generatePresignedUrl(request)
   }
@@ -136,13 +130,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
   def doesObjectExist(bucket: S3Bucket, key: String) = {
     clientFor(bucket).doesObjectExist(bucket.bucket, key)
   }
-
-  def getObject(bucket: S3Bucket, url: URI): model.S3Object = { // TODO why can't this just be by bucket + key to remove end point knowledge
-    // get path and remove leading `/`
-    val key: Key = url.getPath.drop(1)
-    clientFor(bucket).getObject(new GetObjectRequest(bucket.bucket, key))
-  }
-
+  
   def getObject(bucket: S3Bucket, key: String): model.S3Object = {
     clientFor(bucket).getObject(new GetObjectRequest(bucket.bucket, key))
   }
@@ -320,6 +308,7 @@ object S3Ops {
 
     config.withAWSCredentials(builder, localstackAware, maybeRegionOverride).build()
   }
+
 }
 
 object S3 {
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
new file mode 100644
index 0000000000..37441bb811
--- /dev/null
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
@@ -0,0 +1,19 @@
+package com.gu.mediaservice.lib.aws
+
+import com.gu.mediaservice.lib.logging.GridLogging
+
+import java.net.URI
+
+trait S3KeyFromURL extends GridLogging {
+
+  def keyFromS3URL(bucket: S3Bucket, url: URI): String = {
+    val key = if (bucket.endpoint == "10.0.45.121:32090") {
+      url.getPath.drop(bucket.bucket.length + 1)
+    } else {
+      url.getPath.drop(2)
+    }
+    logger.info("Key from bucket " + bucket + " URL " + url + ": " + key)
+    key
+  }
+
+}
diff --git a/cropper/app/lib/CropStore.scala b/cropper/app/lib/CropStore.scala
index 60b3a2672e..a3a4b32682 100644
--- a/cropper/app/lib/CropStore.scala
+++ b/cropper/app/lib/CropStore.scala
@@ -4,10 +4,11 @@ import java.io.File
 import java.net.{URI, URL}
 import scala.concurrent.Future
 import com.gu.mediaservice.lib.S3ImageStorage
+import com.gu.mediaservice.lib.aws.S3KeyFromURL
 import com.gu.mediaservice.lib.logging.LogMarker
 import com.gu.mediaservice.model._
 
-class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropSpecMetadata {
+class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropSpecMetadata with S3KeyFromURL {
   import com.gu.mediaservice.lib.formatting._
 
   def getSecureCropUri(uri: URI): Option[URL] =
@@ -49,9 +50,11 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS
             date           = userMetadata.get("date").flatMap(parseDateTime)
             dimensions     = Dimensions(width, height)
 
+            key = keyFromS3URL(config.imgPublishingBucket, s3Object.uri)
+
             sizing         =
               Asset(
-                signedCropAssetUrl(s3Object.uri),
+                signedCropAssetUrl(key),
                 Some(s3Object.size),
                 objectMetadata.contentType,
                 Some(dimensions),
@@ -82,8 +85,8 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS
     instance.id + "/" + id
   }
 
-  private def signedCropAssetUrl(uri: URI): URI = {
-    signUrlTony(config.imgPublishingBucket, uri).toURI
+  private def signedCropAssetUrl(key: String): URI = {
+    signUrlTony(config.imgPublishingBucket, key).toURI
   }
 
 }
diff --git a/cropper/app/lib/Crops.scala b/cropper/app/lib/Crops.scala
index 33ba37ac9b..404aa5f856 100644
--- a/cropper/app/lib/Crops.scala
+++ b/cropper/app/lib/Crops.scala
@@ -3,7 +3,7 @@ package lib
 import java.io.File
 import com.gu.mediaservice.lib.metadata.FileMetadataHelper
 import com.gu.mediaservice.lib.Files
-import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket, S3KeyFromURL}
 import com.gu.mediaservice.lib.imaging.{ExportResult, ImageOperations}
 import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch}
 import com.gu.mediaservice.model._
@@ -17,7 +17,7 @@ case object InvalidCropRequest extends Exception("Crop request invalid for image
 
 case class MasterCrop(sizing: Future[Asset], file: File, dimensions: Dimensions, aspectRatio: Float)
 
-class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: S3Bucket)(implicit ec: ExecutionContext) extends GridLogging {
+class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: S3Bucket)(implicit ec: ExecutionContext) extends GridLogging with S3KeyFromURL {
   import Files._
 
   private val cropQuality = 75d
@@ -117,7 +117,9 @@ class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOpera
     val hasAlpha = apiImage.fileMetadata.colourModelInformation.get("hasAlpha").flatMap(a => Try(a.toBoolean).toOption).getOrElse(true)
     val cropType = Crops.cropType(mimeType, colourType, hasAlpha)
 
-    val secureUrl = s3.signUrlTony(imageBucket, secureFile)
+    val key = keyFromS3URL(imageBucket, secureFile)
+
+    val secureUrl = s3.signUrlTony(imageBucket, key)
 
     Stopwatch.async(s"making crop assets for ${apiImage.id} ${Crop.getCropId(source.bounds)}") {
       for {
diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala
index f8812aee3a..3394f7e46f 100644
--- a/media-api/app/controllers/MediaApi.scala
+++ b/media-api/app/controllers/MediaApi.scala
@@ -9,7 +9,8 @@ import com.gu.mediaservice.lib.auth.Authentication.{Request, _}
 import com.gu.mediaservice.lib.auth.Permissions.{ArchiveImages, DeleteCropsOrUsages, EditMetadata, UploadImages, DeleteImage => DeleteImagePermission}
 import com.gu.mediaservice.lib.auth._
 import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider
-import com.gu.mediaservice.lib.aws.{ContentDisposition, S3, ThrallMessageSender, UpdateMessage}
+import com.gu.mediaservice.lib.aws.{ContentDisposition, ThrallMessageSender, UpdateMessage}
+import com.gu.mediaservice.lib.aws.{ContentDisposition, S3, S3KeyFromURL, ThrallMessageSender, UpdateMessage}
 import com.gu.mediaservice.lib.config.InstanceForRequest
 import com.gu.mediaservice.lib.events.UsageEvents
 import com.gu.mediaservice.lib.formatting.printDateTime
@@ -45,7 +46,7 @@ class MediaApi(
                 ws: WSClient,
                 authorisation: Authorisation,
                 events: UsageEvents,
-)(implicit val ec: ExecutionContext) extends BaseController with MessageSubjects with ArgoHelpers with ContentDisposition with InstanceForRequest {
+)(implicit val ec: ExecutionContext) extends BaseController with MessageSubjects with ArgoHelpers with ContentDisposition with InstanceForRequest with S3KeyFromURL {
 
   private val gridClient: GridClient = GridClient(config.services)(ws)
 
@@ -250,7 +251,8 @@ class MediaApi(
         val maybeResult = for {
           export <- source.exports.find(_.id.contains(exportId))
           asset <- export.assets.find(_.dimensions.exists(_.width == width))
-          s3Object <- Try(s3Client.getObject(config.imgPublishingBucket, asset.file)).toOption
+          key = keyFromS3URL(config.imgPublishingBucket, asset.file)
+          s3Object <- Try(s3Client.getObject(config.imgPublishingBucket, key)).toOption
           file = StreamConverters.fromInputStream(() => s3Object.getObjectContent)
           entity = HttpEntity.Streamed(file, asset.size, asset.mimeType.map(_.name))
           result = Result(ResponseHeader(OK), entity).withHeaders("Content-Disposition" -> getContentDisposition(source, export, asset, config.shortenDownloadFilename))
@@ -357,7 +359,8 @@ class MediaApi(
         val apiKey = request.user.accessor
         logger.info(s"Download original image: $id from user: ${Authentication.getIdentity(request.user)}", apiKey, id)
         mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OriginalDownloadType)
-        val s3Object = s3Client.getObject(config.imageBucket, image.source.file)
+        val key = keyFromS3URL(config.imageBucket, image.source.file)
+        val s3Object = s3Client.getObject(config.imageBucket, key)
         val file = StreamConverters.fromInputStream(() => s3Object.getObjectContent)
         val entity = HttpEntity.Streamed(file, image.source.size, image.source.mimeType.map(_.name))
 
@@ -410,8 +413,9 @@ class MediaApi(
         logger.info(s"Download optimised image: $id from user: ${Authentication.getIdentity(request.user)}", apiKey, id)
         mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OptimisedDownloadType)
 
+        val key = keyFromS3URL(config.imageBucket, image.optimisedPng.getOrElse(image.source).file)
         val sourceImageUri =
-          new URI(s3Client.signUrl(config.imageBucket, image.optimisedPng.getOrElse(image.source).file, image, imageType = image.optimisedPng match {
+          new URI(s3Client.signUrl(config.imageBucket, key, image, imageType = image.optimisedPng match {
             case Some(_) => OptimisedPng
             case _ => Source
           }))
diff --git a/media-api/app/lib/ImageResponse.scala b/media-api/app/lib/ImageResponse.scala
index 09cfeb1965..5ef908902b 100644
--- a/media-api/app/lib/ImageResponse.scala
+++ b/media-api/app/lib/ImageResponse.scala
@@ -2,6 +2,7 @@ package lib
 
 import com.gu.mediaservice.lib.argo.model._
 import com.gu.mediaservice.lib.auth.{Internal, Tier}
+import com.gu.mediaservice.lib.aws.S3KeyFromURL
 import com.gu.mediaservice.lib.collections.CollectionsManager
 import com.gu.mediaservice.lib.logging.GridLogging
 import com.gu.mediaservice.model._
@@ -21,7 +22,7 @@ import scala.annotation.tailrec
 import scala.util.{Failure, Try}
 
 class ImageResponse(config: MediaApiConfig, s3Client: S3Client, usageQuota: UsageQuota)
-  extends EditsResponse with GridLogging {
+  extends EditsResponse with GridLogging with S3KeyFromURL {
 
   implicit val usageQuotas: UsageQuota = usageQuota
 
@@ -75,11 +76,12 @@ class ImageResponse(config: MediaApiConfig, s3Client: S3Client, usageQuota: Usag
 
     val fileUri = image.source.file
 
-    val imageUrl = s3Client.signUrl(config.imageBucket, fileUri, image, imageType = Source)
+    val key = keyFromS3URL(config.imageBucket, fileUri)
+    val imageUrl = s3Client.signUrl(config.imageBucket, key, image, imageType = Source)
     val pngUrl: Option[String] = pngFileUri
-      .map(s3Client.signUrl(config.imageBucket, _, image, imageType = OptimisedPng))
+      .map(uri => s3Client.signUrl(config.imageBucket, keyFromS3URL(config.imageBucket, uri), image, imageType = OptimisedPng))
 
-    def s3SignedThumbUrl = s3Client.signUrl(config.thumbnailBucket, fileUri, image, imageType = Thumbnail)
+    def s3SignedThumbUrl = s3Client.signUrl(config.thumbnailBucket, key, image, imageType = Thumbnail)
 
     val thumbUrl = config.cloudFrontDomainThumbBucket
       .flatMap(s3Client.signedCloudFrontUrl(_, fileUri.getPath.drop(1)))

From 52edf95052de288402ae628dfb334ec999413741 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 24 Jan 2025 09:36:49 +0000
Subject: [PATCH 17/45] =?UTF-8?q?Local=20S=C2=A3=20urls=20to=20keys.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
index 37441bb811..4b80b16df8 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
@@ -8,9 +8,9 @@ trait S3KeyFromURL extends GridLogging {
 
   def keyFromS3URL(bucket: S3Bucket, url: URI): String = {
     val key = if (bucket.endpoint == "10.0.45.121:32090") {
-      url.getPath.drop(bucket.bucket.length + 1)
+      url.getPath.drop(bucket.bucket.length + 2)
     } else {
-      url.getPath.drop(2)
+      url.getPath.drop(1)
     }
     logger.info("Key from bucket " + bucket + " URL " + url + ": " + key)
     key

From 92422b82fb3991c62ad024342439426166d81a29 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 24 Jan 2025 22:13:08 +0000
Subject: [PATCH 18/45] Local S3 - host.

---
 .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala   | 8 ++++----
 .../scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala  | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index b6df0cfe70..441d3e145b 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -20,7 +20,7 @@ case class S3Object(uri: URI, size: Long, metadata: S3Metadata)
 
 object S3Object {
   private def objectUrl(bucket: String, key: String, s3Endpoint: String): URI = {
-    val bucketUrl = if (s3Endpoint == "10.0.45.121:32090") {
+    val bucketUrl = if (s3Endpoint == "minio.griddev.eelpieconsulting.co.uk") {
       new URI(s"http://$s3Endpoint/$bucket/$key")
     } else {
       val bucketHost = s"$bucket.$s3Endpoint"
@@ -82,7 +82,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     (bucket.endpoint match {
       case "storage.googleapis.com" =>
         googleS3
-      case "10.0.45.121:32090" =>
+      case "minio.griddev.eelpieconsulting.co.uk" =>
         localS3
       case _ =>
         Some(amazonS3)
@@ -130,7 +130,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
   def doesObjectExist(bucket: S3Bucket, key: String) = {
     clientFor(bucket).doesObjectExist(bucket.bucket, key)
   }
-  
+
   def getObject(bucket: S3Bucket, key: String): model.S3Object = {
     clientFor(bucket).getObject(new GetObjectRequest(bucket.bucket, key))
   }
@@ -271,7 +271,7 @@ object S3Ops {
   def buildLocalS3Client(config: CommonConfig): Option[AmazonS3] = {
     config.googleS3AccessKey.flatMap { accessKey =>
       config.googleS3SecretKey.map { secretKey =>
-        val endpointConfig = new EndpointConfiguration("http://10.0.45.121:32090", null)
+        val endpointConfig = new EndpointConfiguration("https://minio.griddev.eelpieconsulting.co.uk", null)
         // create credentials provider
         val credentials = new BasicAWSCredentials(accessKey, secretKey)
         val credentialsProvider = new AWSStaticCredentialsProvider(credentials)
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
index 4b80b16df8..1716b333df 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
@@ -7,7 +7,7 @@ import java.net.URI
 trait S3KeyFromURL extends GridLogging {
 
   def keyFromS3URL(bucket: S3Bucket, url: URI): String = {
-    val key = if (bucket.endpoint == "10.0.45.121:32090") {
+    val key = if (bucket.endpoint == "minio.griddev.eelpieconsulting.co.uk") {
       url.getPath.drop(bucket.bucket.length + 2)
     } else {
       url.getPath.drop(1)

From 4f9168cfbb8ec99d593fdbae40c8b5efa05b9739 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 24 Jan 2025 22:38:11 +0000
Subject: [PATCH 19/45] Imports.

---
 common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index 441d3e145b..977a6310f8 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -9,7 +9,7 @@ import com.amazonaws.{AmazonServiceException, ClientConfiguration}
 import com.gu.mediaservice.lib.config.CommonConfig
 import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch}
 import com.gu.mediaservice.model._
-import org.joda.time.{DateTime, Duration}
+import org.joda.time.DateTime
 
 import java.io.File
 import java.net.{URI, URL}

From 737a87da200122a34bdaceed77fa43a865772483 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sat, 25 Jan 2025 11:50:27 +0000
Subject: [PATCH 20/45] Kahuna UI can connect to ingest bucket endpoint.

---
 kahuna/app/lib/KahunaConfig.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kahuna/app/lib/KahunaConfig.scala b/kahuna/app/lib/KahunaConfig.scala
index eb462c0f80..2c8223431d 100644
--- a/kahuna/app/lib/KahunaConfig.scala
+++ b/kahuna/app/lib/KahunaConfig.scala
@@ -51,7 +51,7 @@ class KahunaConfig(resources: GridConfigResources) extends CommonConfig(resource
   val frameAncestors: Set[String] = getStringSet("security.frameAncestors")
   val connectSources: Set[String] = getStringSet("security.connectSources") ++ maybeIngestBucket.map { ingestBucket =>
     if (isDev) "https://localstack.media.local.dev-gutools.co.uk"
-    else s"https://$ingestBucket.s3.$awsRegion.amazonaws.com"
+    else s"https://${ingestBucket.bucket}.${ingestBucket.endpoint}"
   } ++ telemetryUri
   val fontSources: Set[String] = getStringSet("security.fontSources")
   val imageSources: Set[String] = getStringSet("security.imageSources")

From 613234b47497d9e9e3a9c8f93dff40b3627f8c14 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Wed, 5 Feb 2025 19:14:10 +0000
Subject: [PATCH 21/45] Log levels.

---
 .../main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
index 1716b333df..f1ecb8cbe7 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
@@ -7,13 +7,11 @@ import java.net.URI
 trait S3KeyFromURL extends GridLogging {
 
   def keyFromS3URL(bucket: S3Bucket, url: URI): String = {
-    val key = if (bucket.endpoint == "minio.griddev.eelpieconsulting.co.uk") {
+    if (bucket.endpoint == "minio.griddev.eelpieconsulting.co.uk") {
       url.getPath.drop(bucket.bucket.length + 2)
     } else {
       url.getPath.drop(1)
     }
-    logger.info("Key from bucket " + bucket + " URL " + url + ": " + key)
-    key
   }
 
 }

From db121bac7837c36d8fccfe433fa3b99c30e85426 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Wed, 2 Apr 2025 21:31:33 +0100
Subject: [PATCH 22/45] Noisy log.

---
 image-loader/app/controllers/ImageLoaderController.scala | 2 --
 1 file changed, 2 deletions(-)

diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala
index 192a36cb0b..6ada971220 100644
--- a/image-loader/app/controllers/ImageLoaderController.scala
+++ b/image-loader/app/controllers/ImageLoaderController.scala
@@ -122,8 +122,6 @@ class ImageLoaderController(auth: Authentication,
   }
 
   private def handleMessageFromIngestBucket(sqsMessage: SQSMessage)(basicLogMarker: LogMarker): Future[Unit] = {
-    logger.info(basicLogMarker, sqsMessage.toString)
-
     extractS3KeyFromSqsMessage(sqsMessage) match {
       case Failure(exception) =>
         metrics.failedIngestsFromQueue.increment()

From 8cd3a92f0980df8bc52f476821a18971a94cb845 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Wed, 2 Apr 2025 21:32:05 +0100
Subject: [PATCH 23/45] Noisy log Redact debug.

---
 image-loader/app/lib/imaging/FileMetadataReader.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/image-loader/app/lib/imaging/FileMetadataReader.scala b/image-loader/app/lib/imaging/FileMetadataReader.scala
index f81b9ce3a7..40b9ad413e 100644
--- a/image-loader/app/lib/imaging/FileMetadataReader.scala
+++ b/image-loader/app/lib/imaging/FileMetadataReader.scala
@@ -125,7 +125,7 @@ object FileMetadataReader extends GridLogging {
   val redactionReplacementValue = s"REDACTED (value longer than $redactionThreshold characters, please refer to the metadata stored in the file itself)"
   private def redactLongFieldValues(imageId: String, metadataType: String, exceptions: List[String] = Nil)(props: Map[String, String]) = props.map {
     case (fieldName, value) if value.length > redactionThreshold && !exceptions.exists(fieldName.contains) =>
-      logger.warn(s"Redacting '$fieldName' $metadataType field for image $imageId, as it's problematically long (longer than $redactionThreshold characters")
+      logger.debug(s"Redacting '$fieldName' $metadataType field for image $imageId, as it's problematically long (longer than $redactionThreshold characters")
       fieldName -> redactionReplacementValue
     case keyValuePair => keyValuePair
   }

From 52fde0c5569d2158d0d497b309d9cc11395351c2 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Wed, 2 Apr 2025 21:48:21 +0100
Subject: [PATCH 24/45] Noisy log.

---
 .../scala/com/gu/mediaservice/lib/instances/Instances.scala   | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/instances/Instances.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/instances/Instances.scala
index eafe858fa4..e23f9d208e 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/instances/Instances.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/instances/Instances.scala
@@ -18,9 +18,7 @@ trait Instances extends StrictLogging {
       r.status match {
         case 200 =>
           implicit val ir = Json.reads[Instance]
-          val instances = Json.parse(r.body).as[Seq[Instance]]
-          logger.info("Got instances: " + instances.map(_.id).mkString(","))
-          instances
+          Json.parse(r.body).as[Seq[Instance]]
         case _ =>
           logger.warn("Got non 200 status for instances call: " + r.status)
           Seq.empty

From 9bbb334ac4b74288c9fdb0419ac98705020e6f0c Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 13 Apr 2025 17:17:46 +0100
Subject: [PATCH 25/45] GCP S3 interop does not support S3 bulkDelete;
 reimplement as single calls.

---
 .../lib/ImageIngestOperations.scala           | 49 +++++++++++++------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index f1e6a0f26a..7c4606ca7a 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -59,22 +59,43 @@ class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, co
 
   private def bulkDelete(bucket: S3Bucket, keys: List[String]): Future[Map[String, Boolean]] = keys match {
     case Nil => Future.successful(Map.empty)
-    case _ => Future {
-      try {
-        logger.info(s"Creating S3 bulkDelete request for $bucket / keys: " + keys.mkString(","))
-        deleteObjects(bucket, keys)
-        keys.map { key =>
-          key -> true
-        }.toMap
-      } catch {
-        case partialFailure: MultiObjectDeleteException =>
-          logger.warn(s"Partial failure when deleting images from $bucket: ${partialFailure.getMessage} ${partialFailure.getErrors}")
-          val errorKeys = partialFailure.getErrors.asScala.map(_.getKey).toSet
+    case _ =>
+      val bulkDeleteImplemented = bucket.endpoint != "storage.googleapis.com"
+      if (bulkDeleteImplemented) {
+        Future {
+          try {
+            logger.info(s"Bulk deleting S3 objects from $bucket: " + keys.mkString(","))
+            deleteObjects(bucket, keys)
+            keys.map { key =>
+              key -> true
+            }.toMap
+          } catch {
+            case partialFailure: MultiObjectDeleteException =>
+              logger.warn(s"Partial failure when deleting images from $bucket: ${partialFailure.getMessage} ${partialFailure.getErrors}")
+              val errorKeys = partialFailure.getErrors.asScala.map(_.getKey).toSet
+              keys.map { key =>
+                key -> !errorKeys.contains(key)
+              }.toMap
+          }
+        }
+
+      } else {
+        Future.sequence {
           keys.map { key =>
-            key -> !errorKeys.contains(key)
-          }.toMap
+            Future {
+              logger.info(s"Deleting S3 objects from $bucket: " + key)
+              try {
+                val x = deleteObject(bucket, key)
+                (key, true)
+              } catch {
+                case e: Exception =>
+                  logger.warn(s"Failure when deleting images from $bucket: $key, ${e.getMessage}")
+                  (key, false)
+              }
+            }
+          }
+        }.map(_.toMap)
       }
-    }
   }
 
   def deleteOriginal(id: String)(implicit logMarker: LogMarker, instance: Instance): Future[Unit] = if(isVersionedS3) deleteVersionedImage(imageBucket, fileKeyFromId(id)) else deleteImage(imageBucket, fileKeyFromId(id))

From db6f8437393c0845e90df426d91a753fd29a04fc Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Mon, 28 Apr 2025 21:07:58 +0100
Subject: [PATCH 26/45] Logging tidy up; storeImage

---
 .../src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
index 54b17dba01..a4d603f1a6 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
@@ -17,7 +17,7 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage
   def storeImage(bucket: S3Bucket, id: String, file: File, mimeType: Option[MimeType],
                  meta: Map[String, String] = Map.empty, overwrite: Boolean)
                 (implicit logMarker: LogMarker) = {
-    logger.info(logMarker, s"bucket: $bucket, id: $id, meta: $meta")
+    logger.info(logMarker, s"storeImage to bucket: ${bucket.bucket}, id: $id, meta: $meta")
     val eventualObject = if (overwrite) {
       store(bucket, id, file, mimeType, meta, cacheSetting)
     } else {

From 68a2aea49dff68ed1a7bfca6ae62cfab1c50edc0 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Mon, 28 Apr 2025 21:11:15 +0100
Subject: [PATCH 27/45] Logging tidy up; thumbnail bucket.

---
 .../scala/com/gu/mediaservice/lib/ImageIngestOperations.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index 7c4606ca7a..04a1de029d 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -44,7 +44,7 @@ class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, co
   private def storeThumbnailImage(storableImage: StorableThumbImage)
                                  (implicit logMarker: LogMarker): Future[S3Object] = {
     val instanceSpecificKey = instanceAwareThumbnailImageKey(storableImage)
-    logger.info(s"Storing thumbnail to instance specific key: $thumbnailBucket / $instanceSpecificKey")
+    logger.info(s"Storing thumbnail to instance specific key: ${thumbnailBucket.bucket} / $instanceSpecificKey")
     storeImage(thumbnailBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType),
       overwrite = true)
   }

From bc0b86551a36ecd9819db563a131c127009dbcf6 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Mon, 28 Apr 2025 21:17:05 +0100
Subject: [PATCH 28/45] Clean up; unused unit assignment.

---
 .../scala/com/gu/mediaservice/lib/ImageIngestOperations.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index 04a1de029d..f2758567d1 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -85,7 +85,7 @@ class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, co
             Future {
               logger.info(s"Deleting S3 objects from $bucket: " + key)
               try {
-                val x = deleteObject(bucket, key)
+                deleteObject(bucket, key)
                 (key, true)
               } catch {
                 case e: Exception =>

From a3cc86c61e70469820b3d48adaa60f6f3f1b2139 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Mon, 28 Apr 2025 21:17:47 +0100
Subject: [PATCH 29/45] Logging; lower to debug.

---
 .../scala/com/gu/mediaservice/lib/ImageIngestOperations.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index f2758567d1..0f65f4e7bb 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -89,7 +89,7 @@ class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, co
                 (key, true)
               } catch {
                 case e: Exception =>
-                  logger.warn(s"Failure when deleting images from $bucket: $key, ${e.getMessage}")
+                  logger.debug(s"Failure when deleting images from $bucket: $key, ${e.getMessage}")
                   (key, false)
               }
             }

From 4210b4418ef5357b120c2d3a80e38aa60970e6c0 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Mon, 28 Apr 2025 21:20:17 +0100
Subject: [PATCH 30/45] Logging; bucket name.

---
 .../scala/com/gu/mediaservice/lib/ImageIngestOperations.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index 0f65f4e7bb..d4107b36f0 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -64,7 +64,7 @@ class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, co
       if (bulkDeleteImplemented) {
         Future {
           try {
-            logger.info(s"Bulk deleting S3 objects from $bucket: " + keys.mkString(","))
+            logger.info(s"Bulk deleting S3 objects from ${bucket.bucket}: " + keys.mkString(","))
             deleteObjects(bucket, keys)
             keys.map { key =>
               key -> true

From e81935ada075028d7736dae3626856308b814730 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Mon, 28 Apr 2025 21:21:09 +0100
Subject: [PATCH 31/45] Logging; bucket name.

---
 .../scala/com/gu/mediaservice/lib/ImageIngestOperations.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index d4107b36f0..38c5914746 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -83,7 +83,7 @@ class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, co
         Future.sequence {
           keys.map { key =>
             Future {
-              logger.info(s"Deleting S3 objects from $bucket: " + key)
+              logger.info(s"Deleting S3 objects from ${bucket.bucket}: " + key)
               try {
                 deleteObject(bucket, key)
                 (key, true)

From b84c145ea242ecb39d8dbe950363143a07e2b3a5 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sat, 17 May 2025 10:13:41 +0100
Subject: [PATCH 32/45] Inject S3 into Crops from CropperComponents.

Consistent with all other services.
---
 cropper/app/CropperComponents.scala | 4 +++-
 cropper/app/lib/Crops.scala         | 4 +---
 cropper/test/lib/CropsTest.scala    | 9 +++++----
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/cropper/app/CropperComponents.scala b/cropper/app/CropperComponents.scala
index 509192dedc..d570859ade 100644
--- a/cropper/app/CropperComponents.scala
+++ b/cropper/app/CropperComponents.scala
@@ -1,4 +1,5 @@
 import com.gu.mediaservice.GridClient
+import com.gu.mediaservice.lib.aws.S3
 import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.management.Management
 import com.gu.mediaservice.lib.play.GridComponents
@@ -12,8 +13,9 @@ class CropperComponents(context: Context) extends GridComponents(context, new Cr
 
   val store = new CropStore(config)
   val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath)
+  val s3 = new S3(config)
 
-  val crops = new Crops(config, store, imageOperations, config.imageBucket)
+  val crops = new Crops(config, store, imageOperations, config.imageBucket, s3)
   val notifications = new Notifications(config)
 
   private val gridClient = GridClient(config.services)(wsClient)
diff --git a/cropper/app/lib/Crops.scala b/cropper/app/lib/Crops.scala
index 404aa5f856..6ecfdce9f0 100644
--- a/cropper/app/lib/Crops.scala
+++ b/cropper/app/lib/Crops.scala
@@ -17,7 +17,7 @@ case object InvalidCropRequest extends Exception("Crop request invalid for image
 
 case class MasterCrop(sizing: Future[Asset], file: File, dimensions: Dimensions, aspectRatio: Float)
 
-class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: S3Bucket)(implicit ec: ExecutionContext) extends GridLogging with S3KeyFromURL {
+class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: S3Bucket, s3: S3)(implicit ec: ExecutionContext) extends GridLogging with S3KeyFromURL {
   import Files._
 
   private val cropQuality = 75d
@@ -26,8 +26,6 @@ class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOpera
   // We don't overly care about output crop file sizes here, but prefer a fast output, so turn it right down.
   private val pngCropQuality = 1d
 
-  private val s3 = new S3(config)
-
   def outputFilename(source: SourceImage, bounds: Bounds, outputWidth: Int, fileType: MimeType, isMaster: Boolean = false, instance: Instance): String = {
     val masterString: String = if (isMaster) "master/" else ""
     instance.id + "/" + s"${source.id}/${Crop.getCropId(bounds)}/$masterString$outputWidth${fileType.fileExtension}"
diff --git a/cropper/test/lib/CropsTest.scala b/cropper/test/lib/CropsTest.scala
index 1a15ba9f52..019fcb372b 100644
--- a/cropper/test/lib/CropsTest.scala
+++ b/cropper/test/lib/CropsTest.scala
@@ -59,24 +59,25 @@ class CropsTest extends AnyFunSpec with Matchers with MockitoSugar {
   private val bounds: Bounds = Bounds(10, 20, 30, 40)
   private val outputWidth = 1234
   private val imageBucket = S3Bucket("crops-bucket", S3.AmazonAwsS3Endpoint)
+  private val s3 = new S3(config)
 
   it("should should construct a correct address for a master jpg") {
-    val outputFilename = new Crops(config, store, imageOperations, imageBucket)
+    val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3)
       .outputFilename(source, bounds, outputWidth, Jpeg, isMaster = true, instance = instance)
     outputFilename shouldBe "an-instance/test/10_20_30_40/master/1234.jpg"
   }
   it("should should construct a correct address for a non-master jpg") {
-    val outputFilename = new Crops(config, store, imageOperations, imageBucket)
+    val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3)
       .outputFilename(source, bounds, outputWidth, Jpeg, instance = instance)
     outputFilename shouldBe "an-instance/test/10_20_30_40/1234.jpg"
   }
   it("should should construct a correct address for a non-master tiff") {
-    val outputFilename = new Crops(config, store, imageOperations, imageBucket)
+    val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3)
       .outputFilename(source, bounds, outputWidth, Tiff, instance = instance)
     outputFilename shouldBe "an-instance/test/10_20_30_40/1234.tiff"
   }
   it("should should construct a correct address for a non-master png") {
-    val outputFilename = new Crops(config, store, imageOperations, imageBucket)
+    val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3)
       .outputFilename(source, bounds, outputWidth, Png, instance = instance)
     outputFilename shouldBe "an-instance/test/10_20_30_40/1234.png"
   }

From 03b1291e17485b69e6845c7b4ed93011bfb44a30 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sat, 9 Aug 2025 13:20:45 +0100
Subject: [PATCH 33/45] S3Object takes a full S3Bucket as it's bucket
 parameter. endpoint parameter folds into this.

---
 .../lib/ImageIngestOperations.scala           | 15 ++++++--------
 .../com/gu/mediaservice/lib/aws/S3.scala      | 20 +++++++++----------
 .../test/scala/model/ImageUploadTest.scala    |  2 +-
 3 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index 38c5914746..c158008cfd 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -127,37 +127,34 @@ sealed trait ImageWrapper {
 }
 sealed trait StorableImage extends ImageWrapper {
   def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object(
-    thumbBucket.bucket,
+    thumbBucket,
     ImageIngestOperations.fileKeyFromId(id)(instance),
     file,
     Some(mimeType),
     lastModified = None,
-    meta,
-    s3Endpoint = thumbBucket.endpoint
+    meta
   )
 }
 
 case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage
 case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: DateTime, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage {
   override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object(
-    thumbBucket.bucket,
+    thumbBucket,
     ImageIngestOperations.fileKeyFromId(id)(instance),
     file,
     Some(mimeType),
     lastModified = Some(lastModified),
-    meta,
-    s3Endpoint = thumbBucket.endpoint
+    meta
   )
 }
 case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage {
   override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object(
-    thumbBucket.bucket,
+    thumbBucket,
     ImageIngestOperations.optimisedPngKeyFromId(id)(instance),
     file,
     Some(mimeType),
     lastModified = None,
-    meta = meta,
-    s3Endpoint = thumbBucket.endpoint
+    meta = meta
   )
 }
 
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index 977a6310f8..7ed77dc4e9 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -19,7 +19,8 @@ import scala.concurrent.{ExecutionContext, Future}
 case class S3Object(uri: URI, size: Long, metadata: S3Metadata)
 
 object S3Object {
-  private def objectUrl(bucket: String, key: String, s3Endpoint: String): URI = {
+  private def objectUrl(bucket: S3Bucket, key: String): URI = {
+    val s3Endpoint = bucket.endpoint
     val bucketUrl = if (s3Endpoint == "minio.griddev.eelpieconsulting.co.uk") {
       new URI(s"http://$s3Endpoint/$bucket/$key")
     } else {
@@ -29,11 +30,11 @@ object S3Object {
     bucketUrl
   }
 
-  def apply(bucket: String, key: String, size: Long, metadata: S3Metadata, s3Endpoint: String): S3Object =
-    apply(objectUrl(bucket, key, s3Endpoint), size, metadata)
+  def apply(bucket: S3Bucket, key: String, size: Long, metadata: S3Metadata): S3Object =
+    apply(objectUrl(bucket, key), size, metadata)
 
-  def apply(bucket: String, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime],
-            meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None, s3Endpoint: String): S3Object = {
+  def apply(bucket: S3Bucket, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime],
+            meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None): S3Object = {
     S3Object(
       bucket,
       key,
@@ -45,8 +46,7 @@ object S3Object {
           cacheControl,
           lastModified
         )
-      ),
-      s3Endpoint
+      )
     )
   }
 }
@@ -199,7 +199,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
         client.putObject(req)
         // once we've completed the PUT read back to ensure that we are returning reality
         val metadata = client.getObjectMetadata(bucket.bucket, id)
-        S3Object(bucket.bucket, id, metadata.getContentLength, S3Metadata(metadata), bucket.endpoint)
+        S3Object(bucket, id, metadata.getContentLength, S3Metadata(metadata))
       }(markers)
     }
 
@@ -213,7 +213,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
     }.flatMap {
       case Some(objectMetadata) =>
         logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket $bucket")
-        Future.successful(S3Object(bucket.bucket, id, objectMetadata.getContentLength, S3Metadata(objectMetadata), bucket.endpoint))
+        Future.successful(S3Object(bucket, id, objectMetadata.getContentLength, S3Metadata(objectMetadata)))
       case None =>
         store(bucket, id, file, mimeType, meta, cacheControl)
     }
@@ -227,7 +227,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with
       val summaries = listing.getObjectSummaries.asScala
       summaries.map(summary => (summary.getKey, summary)).foldLeft(List[S3Object]()) {
         case (memo: List[S3Object], (key: String, summary: S3ObjectSummary)) =>
-          S3Object(bucket.bucket, key, summary.getSize, getMetadata(bucket, key), bucket.endpoint) :: memo
+          S3Object(bucket, key, summary.getSize, getMetadata(bucket, key)) :: memo
       }
     }
 
diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala
index 1b76df6239..15625d2223 100644
--- a/image-loader/test/scala/model/ImageUploadTest.scala
+++ b/image-loader/test/scala/model/ImageUploadTest.scala
@@ -53,7 +53,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
 
     def mockStore = (a: StorableImage) =>
       Future.successful(
-        S3Object("madeupname", "madeupkey", a.file, Some(a.mimeType), None, a.meta, None, S3.AmazonAwsS3Endpoint)
+        S3Object(S3Bucket("madeupname", S3.AmazonAwsS3Endpoint), "madeupkey", a.file, Some(a.mimeType), None, a.meta, None)
       )
 
     def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore

From d6d483e42750fb9c42e72de4f28e4533b214068c Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sat, 9 Aug 2025 21:23:58 +0100
Subject: [PATCH 34/45] Incorrect bucket URL.

---
 .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala      | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index 7ed77dc4e9..2d80bb436c 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -21,10 +21,11 @@ case class S3Object(uri: URI, size: Long, metadata: S3Metadata)
 object S3Object {
   private def objectUrl(bucket: S3Bucket, key: String): URI = {
     val s3Endpoint = bucket.endpoint
+    val bucketName = bucket.bucket
     val bucketUrl = if (s3Endpoint == "minio.griddev.eelpieconsulting.co.uk") {
-      new URI(s"http://$s3Endpoint/$bucket/$key")
+      new URI(s"http://$s3Endpoint/$bucketName/$key")
     } else {
-      val bucketHost = s"$bucket.$s3Endpoint"
+      val bucketHost = s"$bucketName.$s3Endpoint"
       new URI("http", bucketHost, s"/$key", null)
     }
     bucketUrl

From 06eeb563f323b114a2d7ac62b03bfa1fb96d803c Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 14 Sep 2025 12:55:36 +0100
Subject: [PATCH 35/45] S3 objectUrl for bucket and key can move to the bucket
 (which knows it's own URL scheme).

---
 .../scala/com/gu/mediaservice/lib/aws/S3.scala  | 13 +------------
 .../com/gu/mediaservice/lib/aws/S3Bucket.scala  | 17 ++++++++++++++++-
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index 2d80bb436c..628b4db8b7 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -19,20 +19,9 @@ import scala.concurrent.{ExecutionContext, Future}
 case class S3Object(uri: URI, size: Long, metadata: S3Metadata)
 
 object S3Object {
-  private def objectUrl(bucket: S3Bucket, key: String): URI = {
-    val s3Endpoint = bucket.endpoint
-    val bucketName = bucket.bucket
-    val bucketUrl = if (s3Endpoint == "minio.griddev.eelpieconsulting.co.uk") {
-      new URI(s"http://$s3Endpoint/$bucketName/$key")
-    } else {
-      val bucketHost = s"$bucketName.$s3Endpoint"
-      new URI("http", bucketHost, s"/$key", null)
-    }
-    bucketUrl
-  }
 
   def apply(bucket: S3Bucket, key: String, size: Long, metadata: S3Metadata): S3Object =
-    apply(objectUrl(bucket, key), size, metadata)
+    apply(bucket.objectUrl(key), size, metadata)
 
   def apply(bucket: S3Bucket, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime],
             meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None): S3Object = {
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
index 2fc155c89c..6b79669091 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
@@ -1,3 +1,18 @@
 package com.gu.mediaservice.lib.aws
 
-case class S3Bucket(bucket: String, endpoint: String)
+import java.net.URI
+
+case class S3Bucket(bucket: String, endpoint: String) {
+  def objectUrl(key: String): URI = {
+    val s3Endpoint = endpoint
+    val bucketName = bucket
+    val bucketUrl = if (s3Endpoint == "minio.griddev.eelpieconsulting.co.uk") {
+      new URI(s"http://$s3Endpoint/$bucketName/$key")
+    } else {
+      val bucketHost = s"$bucketName.$s3Endpoint"
+      new URI("http", bucketHost, s"/$key", null)
+    }
+    bucketUrl
+  }
+
+}

From 49e55bfb703472caf35a2fdedd9e1ea82c41a09a Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 14 Sep 2025 14:58:49 +0100
Subject: [PATCH 36/45] S3KeyFromURL move to S3Bucket as it know's it's own URL
 scheme.

---
 .../com/gu/mediaservice/lib/aws/S3Bucket.scala  |  8 ++++++++
 .../gu/mediaservice/lib/aws/S3KeyFromURL.scala  | 17 -----------------
 cropper/app/lib/CropStore.scala                 |  5 ++---
 cropper/app/lib/Crops.scala                     |  7 +++----
 media-api/app/controllers/MediaApi.scala        | 12 ++++++------
 media-api/app/lib/ImageResponse.scala           |  7 +++----
 6 files changed, 22 insertions(+), 34 deletions(-)
 delete mode 100644 common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
index 6b79669091..d68b4c07eb 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
@@ -15,4 +15,12 @@ case class S3Bucket(bucket: String, endpoint: String) {
     bucketUrl
   }
 
+  def keyFromS3URL(url: URI): String = {
+    if (endpoint == "minio.griddev.eelpieconsulting.co.uk") {
+      url.getPath.drop(bucket.length + 2)
+    } else {
+      url.getPath.drop(1)
+    }
+  }
+
 }
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
deleted file mode 100644
index f1ecb8cbe7..0000000000
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3KeyFromURL.scala
+++ /dev/null
@@ -1,17 +0,0 @@
-package com.gu.mediaservice.lib.aws
-
-import com.gu.mediaservice.lib.logging.GridLogging
-
-import java.net.URI
-
-trait S3KeyFromURL extends GridLogging {
-
-  def keyFromS3URL(bucket: S3Bucket, url: URI): String = {
-    if (bucket.endpoint == "minio.griddev.eelpieconsulting.co.uk") {
-      url.getPath.drop(bucket.bucket.length + 2)
-    } else {
-      url.getPath.drop(1)
-    }
-  }
-
-}
diff --git a/cropper/app/lib/CropStore.scala b/cropper/app/lib/CropStore.scala
index a3a4b32682..034790420f 100644
--- a/cropper/app/lib/CropStore.scala
+++ b/cropper/app/lib/CropStore.scala
@@ -4,11 +4,10 @@ import java.io.File
 import java.net.{URI, URL}
 import scala.concurrent.Future
 import com.gu.mediaservice.lib.S3ImageStorage
-import com.gu.mediaservice.lib.aws.S3KeyFromURL
 import com.gu.mediaservice.lib.logging.LogMarker
 import com.gu.mediaservice.model._
 
-class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropSpecMetadata with S3KeyFromURL {
+class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropSpecMetadata {
   import com.gu.mediaservice.lib.formatting._
 
   def getSecureCropUri(uri: URI): Option[URL] =
@@ -50,7 +49,7 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS
             date           = userMetadata.get("date").flatMap(parseDateTime)
             dimensions     = Dimensions(width, height)
 
-            key = keyFromS3URL(config.imgPublishingBucket, s3Object.uri)
+            key = config.imgPublishingBucket.keyFromS3URL(s3Object.uri)
 
             sizing         =
               Asset(
diff --git a/cropper/app/lib/Crops.scala b/cropper/app/lib/Crops.scala
index 6ecfdce9f0..be3481c014 100644
--- a/cropper/app/lib/Crops.scala
+++ b/cropper/app/lib/Crops.scala
@@ -3,7 +3,7 @@ package lib
 import java.io.File
 import com.gu.mediaservice.lib.metadata.FileMetadataHelper
 import com.gu.mediaservice.lib.Files
-import com.gu.mediaservice.lib.aws.{S3, S3Bucket, S3KeyFromURL}
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
 import com.gu.mediaservice.lib.imaging.{ExportResult, ImageOperations}
 import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch}
 import com.gu.mediaservice.model._
@@ -17,7 +17,7 @@ case object InvalidCropRequest extends Exception("Crop request invalid for image
 
 case class MasterCrop(sizing: Future[Asset], file: File, dimensions: Dimensions, aspectRatio: Float)
 
-class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: S3Bucket, s3: S3)(implicit ec: ExecutionContext) extends GridLogging with S3KeyFromURL {
+class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: S3Bucket, s3: S3)(implicit ec: ExecutionContext) extends GridLogging {
   import Files._
 
   private val cropQuality = 75d
@@ -115,8 +115,7 @@ class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOpera
     val hasAlpha = apiImage.fileMetadata.colourModelInformation.get("hasAlpha").flatMap(a => Try(a.toBoolean).toOption).getOrElse(true)
     val cropType = Crops.cropType(mimeType, colourType, hasAlpha)
 
-    val key = keyFromS3URL(imageBucket, secureFile)
-
+    val key = imageBucket.keyFromS3URL(secureFile)
     val secureUrl = s3.signUrlTony(imageBucket, key)
 
     Stopwatch.async(s"making crop assets for ${apiImage.id} ${Crop.getCropId(source.bounds)}") {
diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala
index 3394f7e46f..d7fe044183 100644
--- a/media-api/app/controllers/MediaApi.scala
+++ b/media-api/app/controllers/MediaApi.scala
@@ -10,7 +10,7 @@ import com.gu.mediaservice.lib.auth.Permissions.{ArchiveImages, DeleteCropsOrUsa
 import com.gu.mediaservice.lib.auth._
 import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider
 import com.gu.mediaservice.lib.aws.{ContentDisposition, ThrallMessageSender, UpdateMessage}
-import com.gu.mediaservice.lib.aws.{ContentDisposition, S3, S3KeyFromURL, ThrallMessageSender, UpdateMessage}
+import com.gu.mediaservice.lib.aws.{ContentDisposition, S3, ThrallMessageSender, UpdateMessage}
 import com.gu.mediaservice.lib.config.InstanceForRequest
 import com.gu.mediaservice.lib.events.UsageEvents
 import com.gu.mediaservice.lib.formatting.printDateTime
@@ -46,7 +46,7 @@ class MediaApi(
                 ws: WSClient,
                 authorisation: Authorisation,
                 events: UsageEvents,
-)(implicit val ec: ExecutionContext) extends BaseController with MessageSubjects with ArgoHelpers with ContentDisposition with InstanceForRequest with S3KeyFromURL {
+)(implicit val ec: ExecutionContext) extends BaseController with MessageSubjects with ArgoHelpers with ContentDisposition with InstanceForRequest {
 
   private val gridClient: GridClient = GridClient(config.services)(ws)
 
@@ -251,8 +251,8 @@ class MediaApi(
         val maybeResult = for {
           export <- source.exports.find(_.id.contains(exportId))
           asset <- export.assets.find(_.dimensions.exists(_.width == width))
-          key = keyFromS3URL(config.imgPublishingBucket, asset.file)
-          s3Object <- Try(s3Client.getObject(config.imgPublishingBucket, key)).toOption
+          key = config.imgPublishingBucket.keyFromS3URL(asset.file)
+          s3Object <- Try(s3Client.getObject(config.imgPublishingBucket, key)).toOption // TODO raw S3 client usage!
           file = StreamConverters.fromInputStream(() => s3Object.getObjectContent)
           entity = HttpEntity.Streamed(file, asset.size, asset.mimeType.map(_.name))
           result = Result(ResponseHeader(OK), entity).withHeaders("Content-Disposition" -> getContentDisposition(source, export, asset, config.shortenDownloadFilename))
@@ -359,7 +359,7 @@ class MediaApi(
         val apiKey = request.user.accessor
         logger.info(s"Download original image: $id from user: ${Authentication.getIdentity(request.user)}", apiKey, id)
         mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OriginalDownloadType)
-        val key = keyFromS3URL(config.imageBucket, image.source.file)
+        val key = config.imageBucket.keyFromS3URL(image.source.file)
         val s3Object = s3Client.getObject(config.imageBucket, key)
         val file = StreamConverters.fromInputStream(() => s3Object.getObjectContent)
         val entity = HttpEntity.Streamed(file, image.source.size, image.source.mimeType.map(_.name))
@@ -413,7 +413,7 @@ class MediaApi(
         logger.info(s"Download optimised image: $id from user: ${Authentication.getIdentity(request.user)}", apiKey, id)
         mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OptimisedDownloadType)
 
-        val key = keyFromS3URL(config.imageBucket, image.optimisedPng.getOrElse(image.source).file)
+        val key = config.imageBucket.keyFromS3URL(image.optimisedPng.getOrElse(image.source).file)
         val sourceImageUri =
           new URI(s3Client.signUrl(config.imageBucket, key, image, imageType = image.optimisedPng match {
             case Some(_) => OptimisedPng
diff --git a/media-api/app/lib/ImageResponse.scala b/media-api/app/lib/ImageResponse.scala
index 5ef908902b..8fff479bee 100644
--- a/media-api/app/lib/ImageResponse.scala
+++ b/media-api/app/lib/ImageResponse.scala
@@ -2,7 +2,6 @@ package lib
 
 import com.gu.mediaservice.lib.argo.model._
 import com.gu.mediaservice.lib.auth.{Internal, Tier}
-import com.gu.mediaservice.lib.aws.S3KeyFromURL
 import com.gu.mediaservice.lib.collections.CollectionsManager
 import com.gu.mediaservice.lib.logging.GridLogging
 import com.gu.mediaservice.model._
@@ -22,7 +21,7 @@ import scala.annotation.tailrec
 import scala.util.{Failure, Try}
 
 class ImageResponse(config: MediaApiConfig, s3Client: S3Client, usageQuota: UsageQuota)
-  extends EditsResponse with GridLogging with S3KeyFromURL {
+  extends EditsResponse with GridLogging {
 
   implicit val usageQuotas: UsageQuota = usageQuota
 
@@ -76,10 +75,10 @@ class ImageResponse(config: MediaApiConfig, s3Client: S3Client, usageQuota: Usag
 
     val fileUri = image.source.file
 
-    val key = keyFromS3URL(config.imageBucket, fileUri)
+    val key = config.imageBucket.keyFromS3URL(fileUri)
     val imageUrl = s3Client.signUrl(config.imageBucket, key, image, imageType = Source)
     val pngUrl: Option[String] = pngFileUri
-      .map(uri => s3Client.signUrl(config.imageBucket, keyFromS3URL(config.imageBucket, uri), image, imageType = OptimisedPng))
+      .map(uri => s3Client.signUrl(config.imageBucket, config.imageBucket.keyFromS3URL(uri), image, imageType = OptimisedPng))
 
     def s3SignedThumbUrl = s3Client.signUrl(config.thumbnailBucket, key, image, imageType = Thumbnail)
 

From 7b0a6c5b7d4e77309b9a963b53982910dc4cc7fe Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 14 Sep 2025 15:06:09 +0100
Subject: [PATCH 37/45] Clean up; further isolating S3Client as the cloudfront
 concern.

---
 media-api/app/MediaApiComponents.scala   | 5 +++--
 media-api/app/controllers/MediaApi.scala | 8 ++++----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/media-api/app/MediaApiComponents.scala b/media-api/app/MediaApiComponents.scala
index e84b796bb0..bbbc4e0d15 100644
--- a/media-api/app/MediaApiComponents.scala
+++ b/media-api/app/MediaApiComponents.scala
@@ -1,4 +1,4 @@
-import com.gu.mediaservice.lib.aws.ThrallMessageSender
+import com.gu.mediaservice.lib.aws.{S3, ThrallMessageSender}
 import com.gu.mediaservice.lib.instances.InstancesClient
 import com.gu.mediaservice.lib.management.{ElasticSearchHealthCheck, Management}
 import com.gu.mediaservice.lib.metadata.SoftDeletedMetadataTable
@@ -18,6 +18,7 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M
   val mediaApiMetrics = new MediaApiMetrics(config, actorSystem, applicationLifecycle)
 
   val s3Client = new S3Client(config)
+  val s3 = new S3(config)
 
   val usageQuota = new UsageQuota(config, actorSystem.scheduler)
   usageQuota.quotaStore.update()
@@ -31,7 +32,7 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M
 
   val softDeletedMetadataTable = new SoftDeletedMetadataTable(config)
 
-  val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, controllerComponents, s3Client, mediaApiMetrics, wsClient, authorisation, events)
+  val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, controllerComponents, s3, mediaApiMetrics, wsClient, authorisation, events)
   val suggestionController = new SuggestionController(auth, elasticSearch, controllerComponents)
   val aggController = new AggregationController(auth, elasticSearch, controllerComponents)
   val usageController = new UsageController(auth, config, elasticSearch, usageQuota, controllerComponents)
diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala
index d7fe044183..e566bc4ad6 100644
--- a/media-api/app/controllers/MediaApi.scala
+++ b/media-api/app/controllers/MediaApi.scala
@@ -41,7 +41,7 @@ class MediaApi(
                 imageResponse: ImageResponse,
                 config: MediaApiConfig,
                 override val controllerComponents: ControllerComponents,
-                s3Client: S3Client,
+                s3: S3,
                 mediaApiMetrics: MediaApiMetrics,
                 ws: WSClient,
                 authorisation: Authorisation,
@@ -252,7 +252,7 @@ class MediaApi(
           export <- source.exports.find(_.id.contains(exportId))
           asset <- export.assets.find(_.dimensions.exists(_.width == width))
           key = config.imgPublishingBucket.keyFromS3URL(asset.file)
-          s3Object <- Try(s3Client.getObject(config.imgPublishingBucket, key)).toOption // TODO raw S3 client usage!
+          s3Object <- Try(s3.getObject(config.imgPublishingBucket, key)).toOption
           file = StreamConverters.fromInputStream(() => s3Object.getObjectContent)
           entity = HttpEntity.Streamed(file, asset.size, asset.mimeType.map(_.name))
           result = Result(ResponseHeader(OK), entity).withHeaders("Content-Disposition" -> getContentDisposition(source, export, asset, config.shortenDownloadFilename))
@@ -360,7 +360,7 @@ class MediaApi(
         logger.info(s"Download original image: $id from user: ${Authentication.getIdentity(request.user)}", apiKey, id)
         mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OriginalDownloadType)
         val key = config.imageBucket.keyFromS3URL(image.source.file)
-        val s3Object = s3Client.getObject(config.imageBucket, key)
+        val s3Object = s3.getObject(config.imageBucket, key)
         val file = StreamConverters.fromInputStream(() => s3Object.getObjectContent)
         val entity = HttpEntity.Streamed(file, image.source.size, image.source.mimeType.map(_.name))
 
@@ -415,7 +415,7 @@ class MediaApi(
 
         val key = config.imageBucket.keyFromS3URL(image.optimisedPng.getOrElse(image.source).file)
         val sourceImageUri =
-          new URI(s3Client.signUrl(config.imageBucket, key, image, imageType = image.optimisedPng match {
+          new URI(s3.signUrl(config.imageBucket, key, image, imageType = image.optimisedPng match {
             case Some(_) => OptimisedPng
             case _ => Source
           }))

From d5486c31960548a083f4609bcaa15eb7f647e011 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 14 Sep 2025 15:09:01 +0100
Subject: [PATCH 38/45] Refactor; extracting the is path style URLs bucket
 decision.

---
 .../main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
index d68b4c07eb..26024c6711 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
@@ -6,7 +6,7 @@ case class S3Bucket(bucket: String, endpoint: String) {
   def objectUrl(key: String): URI = {
     val s3Endpoint = endpoint
     val bucketName = bucket
-    val bucketUrl = if (s3Endpoint == "minio.griddev.eelpieconsulting.co.uk") {
+    val bucketUrl = if (isPathStyleURLs(s3Endpoint)) {
       new URI(s"http://$s3Endpoint/$bucketName/$key")
     } else {
       val bucketHost = s"$bucketName.$s3Endpoint"
@@ -16,11 +16,15 @@ case class S3Bucket(bucket: String, endpoint: String) {
   }
 
   def keyFromS3URL(url: URI): String = {
-    if (endpoint == "minio.griddev.eelpieconsulting.co.uk") {
+    if (isPathStyleURLs(endpoint)) {
       url.getPath.drop(bucket.length + 2)
     } else {
       url.getPath.drop(1)
     }
   }
 
+  private def isPathStyleURLs(s3Endpoint: String) = {
+    s3Endpoint == "minio.griddev.eelpieconsulting.co.uk"
+  }
+
 }

From d22e9e60d5722e4e7f1a36f979f1b849b30c6895 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 14 Sep 2025 15:17:07 +0100
Subject: [PATCH 39/45] S3 path based URLs decision moves to bucket property
 and config.

---
 .../scala/com/gu/mediaservice/lib/aws/S3Bucket.scala   | 10 +++-------
 .../com/gu/mediaservice/lib/config/CommonConfig.scala  |  8 ++++----
 cropper/app/lib/CropperConfig.scala                    |  3 ++-
 cropper/test/lib/CropsTest.scala                       |  2 +-
 image-loader/app/lib/ImageLoaderConfig.scala           |  2 +-
 image-loader/test/scala/model/ImageUploadTest.scala    |  4 ++--
 image-loader/test/scala/model/ProjectorTest.scala      |  2 +-
 media-api/app/lib/MediaApiConfig.scala                 |  8 ++++----
 .../auth/provider/ApiKeyAuthenticationProvider.scala   |  2 +-
 .../lib/auth/ApiKeyAuthenticationProviderTest.scala    |  2 +-
 thrall/app/lib/ThrallConfig.scala                      |  2 +-
 11 files changed, 21 insertions(+), 24 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
index 26024c6711..f9ac5b9cc5 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
@@ -2,11 +2,11 @@ package com.gu.mediaservice.lib.aws
 
 import java.net.URI
 
-case class S3Bucket(bucket: String, endpoint: String) {
+case class S3Bucket(bucket: String, endpoint: String, usesPathStyleURLs: Boolean) {
   def objectUrl(key: String): URI = {
     val s3Endpoint = endpoint
     val bucketName = bucket
-    val bucketUrl = if (isPathStyleURLs(s3Endpoint)) {
+    val bucketUrl = if (usesPathStyleURLs) {
       new URI(s"http://$s3Endpoint/$bucketName/$key")
     } else {
       val bucketHost = s"$bucketName.$s3Endpoint"
@@ -16,15 +16,11 @@ case class S3Bucket(bucket: String, endpoint: String) {
   }
 
   def keyFromS3URL(url: URI): String = {
-    if (isPathStyleURLs(endpoint)) {
+    if (usesPathStyleURLs) {
       url.getPath.drop(bucket.length + 2)
     } else {
       url.getPath.drop(1)
     }
   }
 
-  private def isPathStyleURLs(s3Endpoint: String) = {
-    s3Endpoint == "minio.griddev.eelpieconsulting.co.uk"
-  }
-
 }
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
index 3b0b448cbd..acc0edc4dc 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
@@ -57,13 +57,13 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
     ingestBucket <- stringOpt("s3.ingest.bucket.name")
     ingestBucketEndpoint <- stringOpt("s3.ingest.bucket.endpoint")
   } yield {
-    S3Bucket(ingestBucket, ingestBucketEndpoint)
+    S3Bucket(ingestBucket, ingestBucketEndpoint, usesPathStyleURLs = booleanOpt("s3.ingest.bucket.pathStyleURLs").getOrElse(false))
   }
   val maybeFailBucket: Option[S3Bucket] = for {
     failBucket <- stringOpt("s3.fail.bucket.name")
     failBucketEndpoint <- stringOpt("s3.fail.bucket.endpoint")
   } yield {
-    S3Bucket(failBucket, failBucketEndpoint)
+    S3Bucket(failBucket, failBucketEndpoint, usesPathStyleURLs = booleanOpt("s3.fail.bucket.pathStyleURLs").getOrElse(false))
   }
 
   val maybeUploadLimitInBytes: Option[Int] = intOpt("upload.limit.mb").map(_ * 1024 * 1024)
@@ -79,8 +79,8 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
 
   val services = new SingleHostServices(domainRoot)
 
-  val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket.name"), string("s3.image.bucket.endpoint"))
-  val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket.name"), string("s3.thumb.bucket.endpoint"))
+  val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket.name"), string("s3.image.bucket.endpoint"), usesPathStyleURLs = booleanOpt("s3.image.bucket.pathStyleURLs").getOrElse(false))
+  val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket.name"), string("s3.thumb.bucket.endpoint"), usesPathStyleURLs = booleanOpt("s3.thumb.bucket.pathStyleURLs").getOrElse(false))
 
   val googleS3AccessKey = stringOpt("s3.accessKey")
   val googleS3SecretKey = stringOpt("s3.secretKey")
diff --git a/cropper/app/lib/CropperConfig.scala b/cropper/app/lib/CropperConfig.scala
index 395122932c..979824eebb 100644
--- a/cropper/app/lib/CropperConfig.scala
+++ b/cropper/app/lib/CropperConfig.scala
@@ -10,7 +10,8 @@ import java.io.File
 class CropperConfig(resources: GridConfigResources) extends CommonConfig(resources) {
   val imgPublishingBucket: S3Bucket = S3Bucket(
     string("publishing.image.bucket"),
-    S3.AmazonAwsS3Endpoint
+    S3.AmazonAwsS3Endpoint,
+    usesPathStyleURLs = false
   )
   val canDownloadCrop: Boolean = boolean("canDownloadCrop")
 
diff --git a/cropper/test/lib/CropsTest.scala b/cropper/test/lib/CropsTest.scala
index 019fcb372b..77f6c4ace7 100644
--- a/cropper/test/lib/CropsTest.scala
+++ b/cropper/test/lib/CropsTest.scala
@@ -58,7 +58,7 @@ class CropsTest extends AnyFunSpec with Matchers with MockitoSugar {
   private val source: SourceImage = SourceImage("test", mock[Asset], valid = true, mock[ImageMetadata], mock[FileMetadata])
   private val bounds: Bounds = Bounds(10, 20, 30, 40)
   private val outputWidth = 1234
-  private val imageBucket = S3Bucket("crops-bucket", S3.AmazonAwsS3Endpoint)
+  private val imageBucket = S3Bucket("crops-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false)
   private val s3 = new S3(config)
 
   it("should should construct a correct address for a master jpg") {
diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala
index 43312a7448..10cc71952b 100644
--- a/image-loader/app/lib/ImageLoaderConfig.scala
+++ b/image-loader/app/lib/ImageLoaderConfig.scala
@@ -16,7 +16,7 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res
   val maybeImageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket")
 
   val quarantineBucket: Option[S3Bucket] = stringOpt("s3.quarantine.bucket").map { bucket =>
-    S3Bucket(bucket, S3.AmazonAwsS3Endpoint)
+    S3Bucket(bucket, S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false)
   }
   val uploadToQuarantineEnabled: Boolean = boolean("upload.quarantine.enabled")
 
diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala
index 15625d2223..027c2c6a00 100644
--- a/image-loader/test/scala/model/ImageUploadTest.scala
+++ b/image-loader/test/scala/model/ImageUploadTest.scala
@@ -32,7 +32,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
   private implicit val logMarker: MockLogMarker = new MockLogMarker()
     // For mime type info, see https://github.com/guardian/grid/pull/2568
     val tempDir = new File("/tmp")
-    val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), S3Bucket("img-bucket", S3.AmazonAwsS3Endpoint), S3Bucket("thumb-bucket", S3.AmazonAwsS3Endpoint))
+    val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), S3Bucket("img-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false), S3Bucket("thumb-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false))
 
   /**
     * @todo: I flailed about until I found a path that worked, but
@@ -53,7 +53,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
 
     def mockStore = (a: StorableImage) =>
       Future.successful(
-        S3Object(S3Bucket("madeupname", S3.AmazonAwsS3Endpoint), "madeupkey", a.file, Some(a.mimeType), None, a.meta, None)
+        S3Object(S3Bucket("madeupname", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false), "madeupkey", a.file, Some(a.mimeType), None, a.meta, None)
       )
 
     def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore
diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala
index 073949a042..d9748f588d 100644
--- a/image-loader/test/scala/model/ProjectorTest.scala
+++ b/image-loader/test/scala/model/ProjectorTest.scala
@@ -38,7 +38,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc
 
   private val imageOperations = new ImageOperations(ctxPath)
 
-  private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, S3Bucket("img-bucket", S3.AmazonAwsS3Endpoint), S3Bucket("thumb-bucket", S3.AmazonAwsS3Endpoint))
+  private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, S3Bucket("img-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false), S3Bucket("thumb-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false))
 
   private val s3 = mock[S3]
   private val auth = mock[Authentication]
diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala
index ed8466f138..1eb97adef9 100644
--- a/media-api/app/lib/MediaApiConfig.scala
+++ b/media-api/app/lib/MediaApiConfig.scala
@@ -14,17 +14,17 @@ case class StoreConfig(
 )
 
 class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) {
-  val configBucket: S3Bucket = S3Bucket(string("s3.config.bucket"), S3.AmazonAwsS3Endpoint)
-  val usageMailBucket: S3Bucket = S3Bucket(string("s3.usagemail.bucket"), S3.AmazonAwsS3Endpoint)
+  val configBucket: S3Bucket = S3Bucket(string("s3.config.bucket"), S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false)
+  val usageMailBucket: S3Bucket = S3Bucket(string("s3.usagemail.bucket"), S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false)
 
   val quotaStoreKey: String = string("quota.store.key")
   val quotaStoreConfig: StoreConfig = StoreConfig(configBucket, quotaStoreKey)
 
   //Lazy allows this to be empty and not break things unless used somewhere
-  lazy val imgPublishingBucket: S3Bucket = S3Bucket(string("publishing.image.bucket"), S3.AmazonAwsS3Endpoint)
+  lazy val imgPublishingBucket: S3Bucket = S3Bucket(string("publishing.image.bucket"), S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false)
 
   val cloudFrontDomainThumbBucket: Option[String]   = stringOpt("cloudfront.domain.thumbbucket")
-  val cloudFrontPrivateKeyBucket: Option[S3Bucket]    = stringOpt("cloudfront.private-key.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint))
+  val cloudFrontPrivateKeyBucket: Option[S3Bucket]    = stringOpt("cloudfront.private-key.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false))
   val cloudFrontPrivateKeyBucketKey: Option[String] = stringOpt("cloudfront.private-key.key")
   val cloudFrontKeyPairId: Option[String]           = stringOpt("cloudfront.keypair.id")
 
diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala
index 39fd51ac6b..f282bd20b9 100644
--- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala
+++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala
@@ -26,7 +26,7 @@ class ApiKeyAuthenticationProvider(configuration: Configuration, resources: Auth
   var keyStorePlaceholder: Option[KeyStore] = _
 
   override def initialise(): Unit = {
-    val authKeyStoreBucket  = S3Bucket(configuration.get[String]("authKeyStoreBucket"), S3.AmazonAwsS3Endpoint)
+    val authKeyStoreBucket  = S3Bucket(configuration.get[String]("authKeyStoreBucket"), S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false)
     val store = new KeyStore(authKeyStoreBucket, resources.commonConfig)
     store.scheduleUpdates(resources.actorSystem.scheduler)
     keyStorePlaceholder = Some(store)
diff --git a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala
index 220302deee..76c2bfd1b4 100644
--- a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala
+++ b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala
@@ -44,7 +44,7 @@ class ApiKeyAuthenticationProviderTest extends AsyncFreeSpec with Matchers with
       Future.successful(())
     }
 
-    override def keyStore: KeyStore = new KeyStore(S3Bucket("not-used", S3.AmazonAwsS3Endpoint), resources.commonConfig) {
+    override def keyStore: KeyStore = new KeyStore(S3Bucket("not-used", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false), resources.commonConfig) {
       override def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = {
         key match {
           case "key-chuckle" => Some(ApiAccessor("brothers", Internal))
diff --git a/thrall/app/lib/ThrallConfig.scala b/thrall/app/lib/ThrallConfig.scala
index 1a08923f9f..48ab0dd971 100644
--- a/thrall/app/lib/ThrallConfig.scala
+++ b/thrall/app/lib/ThrallConfig.scala
@@ -56,7 +56,7 @@ object KinesisReceiverConfig {
 }
 
 class ThrallConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) {
-  val maybeReaperBucket: Option[S3Bucket] = stringOpt("s3.reaper.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint))
+  val maybeReaperBucket: Option[S3Bucket] = stringOpt("s3.reaper.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false))
   val maybeReaperCountPerRun: Option[Int] = intOpt("reaper.countPerRun")
 
   val metadataTopicArn: String = string("indexed.image.sns.topic.arn")

From 9f5742427290fcc8b46b9d0ee5aad77d7ec8be92 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 30 Mar 2025 11:40:06 +0100
Subject: [PATCH 40/45] Reindex process queues project image operations.

---
 .../gu/mediaservice/model/ThrallMessage.scala |  3 ++
 .../mediaservice/syntax/MessageSubjects.scala |  2 +-
 thrall/app/ThrallComponents.scala             |  3 +-
 thrall/app/controllers/ThrallController.scala | 30 ++++++-------------
 thrall/app/lib/kinesis/MessageProcessor.scala | 27 +++++++++++++++--
 .../app/lib/kinesis/MessageTranslator.scala   |  4 +++
 .../app/lib/kinesis/ThrallEventConsumer.scala |  7 +++--
 7 files changed, 48 insertions(+), 28 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala
index d37d61d388..f7fc7d26dc 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala
@@ -89,6 +89,7 @@ object ExternalThrallMessage{
   implicit val upsertFromProjectionMessage: OFormat[UpsertFromProjectionMessage] = Json.format[UpsertFromProjectionMessage]
 
   implicit val createInstanceMessage: OFormat[CreateInstanceMessage] = Json.format[CreateInstanceMessage]
+  implicit val reindexImageMessage: OFormat[ReindexImageMessage] = Json.format[ReindexImageMessage]
 
   implicit val writes: OWrites[ExternalThrallMessage] = Json.writes[ExternalThrallMessage]
   implicit val reads: Reads[ExternalThrallMessage] = Json.reads[ExternalThrallMessage]
@@ -168,3 +169,5 @@ case class CompleteMigrationMessage(lastModified: DateTime, instance: Instance)
 }
 
 case class CreateInstanceMessage(id: String, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage
+
+case class ReindexImageMessage(id: String, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala b/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala
index c40ff2de17..b0a2f32c6e 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala
@@ -22,7 +22,7 @@ trait MessageSubjects {
   val UpdateImageSyndicationMetadata = "update-image-syndication-metadata"
   val UpdateImagePhotoshootMetadata = "update-image-photoshoot-metadata"
   val CreateInstance = "create-instance"
-
+  val ReindexImage = "reindex-image"
 }
 
 object MessageSubjects extends MessageSubjects
diff --git a/thrall/app/ThrallComponents.scala b/thrall/app/ThrallComponents.scala
index 426b7b5f5d..a6033ab2f7 100644
--- a/thrall/app/ThrallComponents.scala
+++ b/thrall/app/ThrallComponents.scala
@@ -80,7 +80,8 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr
     gridClient,
     auth,
     instanceMessageSender,
-    events
+    events,
+    messageSender
   )
 
   val thrallStreamProcessor = new ThrallStreamProcessor(
diff --git a/thrall/app/controllers/ThrallController.scala b/thrall/app/controllers/ThrallController.scala
index e354416946..bc978b274b 100644
--- a/thrall/app/controllers/ThrallController.scala
+++ b/thrall/app/controllers/ThrallController.scala
@@ -10,8 +10,8 @@ import com.gu.mediaservice.lib.aws.{S3, S3Bucket, ThrallMessageSender, UpdateMes
 import com.gu.mediaservice.lib.config.{InstanceForRequest, Services}
 import com.gu.mediaservice.lib.elasticsearch.{NotRunning, Running}
 import com.gu.mediaservice.lib.logging.GridLogging
-import com.gu.mediaservice.model.{CompleteMigrationMessage, CreateMigrationIndexMessage, Instance, UpsertFromProjectionMessage}
-import com.gu.mediaservice.syntax.MessageSubjects.Image
+import com.gu.mediaservice.model.{CompleteMigrationMessage, CreateMigrationIndexMessage, Instance, ReindexImageMessage, UpsertFromProjectionMessage}
+import com.gu.mediaservice.syntax.MessageSubjects.{Image, ReindexImage}
 import lib.elasticsearch.ElasticSearch
 import lib.{MigrationRequest, OptionalFutureRunner, Paging, ThrallStore}
 import org.joda.time.{DateTime, DateTimeZone}
@@ -328,27 +328,15 @@ class ThrallController(
     val mediaIds = getMediaIdsFromS3(Seq.empty, None)
     logger.info(s"Reindexing ${mediaIds.size} images for instance ${instance.id}")
     mediaIds.foreach { mediaId =>
-      Await.result(reindexImage(mediaId), Duration(10, TimeUnit.SECONDS))
+      messageSender.publish(
+        UpdateMessage(
+          subject = ReindexImage,
+          id = Some(mediaId),
+          instance = instance
+        )
+      )
     }
     Ok("ok")
   }
 
-  private def reindexImage(mediaId: String)(implicit instance: Instance) = {
-    logger.info(s"Reindexing from s3 ${instance.id} / $mediaId")
-
-    gridClient.getImageLoaderProjection(mediaId, auth.innerServiceCall).map { maybeImage =>
-      logger.info(s"Projected ${instance.id} / $mediaId to $maybeImage}")
-      maybeImage.exists { image =>
-        val updateMessage = UpdateMessage(subject = Image, image = Some(image), instance = instance)
-        logger.info(s"Publishing projected image as a thrall image message: ${updateMessage.id}")
-        messageSender.publish(updateMessage)
-        true
-      }
-    }.recover {
-      case _: Throwable =>
-        logger.warn(s"Error while reindexing ${instance.id} / $mediaId - Image has not been reindexed!")
-        false
-    }
-  }
-
 }
diff --git a/thrall/app/lib/kinesis/MessageProcessor.scala b/thrall/app/lib/kinesis/MessageProcessor.scala
index c6e328b62c..f8d0db29c7 100644
--- a/thrall/app/lib/kinesis/MessageProcessor.scala
+++ b/thrall/app/lib/kinesis/MessageProcessor.scala
@@ -2,12 +2,13 @@ package lib.kinesis
 
 import com.gu.mediaservice.GridClient
 import com.gu.mediaservice.lib.auth.Authentication
-import com.gu.mediaservice.lib.aws.EsResponse
+import com.gu.mediaservice.lib.aws.{EsResponse, ThrallMessageSender, UpdateMessage}
 import com.gu.mediaservice.lib.elasticsearch.{ElasticNotFoundException, Running}
 import com.gu.mediaservice.lib.events.UsageEvents
 import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, combineMarkers}
 import com.gu.mediaservice.model.{AddImageLeaseMessage, CreateMigrationIndexMessage, DeleteImageExportsMessage, DeleteImageMessage, DeleteUsagesMessage, ImageMessage, MigrateImageMessage, RemoveImageLeaseMessage, ReplaceImageLeasesMessage, SetImageCollectionsMessage, SoftDeleteImageMessage, ThrallMessage, UnSoftDeleteImageMessage, UpdateImageExportsMessage, UpdateImagePhotoshootMetadataMessage, UpdateImageSyndicationMetadataMessage, UpdateImageUsagesMessage, UpdateImageUserMetadataMessage}
 import com.gu.mediaservice.model.usage.{Usage, UsageNotice}
+import com.gu.mediaservice.syntax.MessageSubjects.Image
 import instances.{InstanceMessageSender, InstanceStatusMessage}
 // import all except `Right`, which otherwise shadows the type used in `Either`s
 import com.gu.mediaservice.model.{Right => _, _}
@@ -33,7 +34,8 @@ class MessageProcessor(
   gridClient: GridClient,
   auth: Authentication,
   instanceMessageSender: InstanceMessageSender,
-  usageEvents: UsageEvents
+  usageEvents: UsageEvents,
+  messageSender: ThrallMessageSender
 ) extends GridLogging with MessageSubjects {
 
   def process(updateMessage: ThrallMessage, logMarker: LogMarker)(implicit ec: ExecutionContext): Future[Any] = {
@@ -60,6 +62,7 @@ class MessageProcessor(
       case message: UpdateUsageStatusMessage => updateUsageStatus(message, logMarker)
       case message: CompleteMigrationMessage => completeMigration(message, logMarker)
       case message: CreateInstanceMessage => setupNewInstance(message, logMarker)
+      case message: ReindexImageMessage => reindexImage(message, logMarker)
       case _ =>
         logger.info(s"Unmatched ThrallMessage type: ${updateMessage.subject}; ignoring")
         Future.successful(())
@@ -267,4 +270,24 @@ class MessageProcessor(
     }
   }
 
+  private def reindexImage(message: ReindexImageMessage, logMarker: LogMarker)(implicit ec: ExecutionContext): Future[Boolean] = {
+    val mediaId = message.id
+    implicit val instance: Instance = message.instance
+    logger.info(s"Reindexing from s3 ${instance.id} / $mediaId")
+
+    gridClient.getImageLoaderProjection(mediaId, auth.innerServiceCall).map { maybeImage =>
+      logger.info(s"Projected ${instance.id} / $mediaId to $maybeImage}")
+      maybeImage.exists { image =>
+        val updateMessage = UpdateMessage(subject = Image, image = Some(image), instance = instance)
+        logger.info(s"Publishing projected image as a thrall image message: ${updateMessage.id}")
+        messageSender.publish(updateMessage)
+        true
+      }
+    }.recover {
+      case _: Throwable =>
+        logger.warn(s"Error while reindexing ${instance.id} / $mediaId - Image has not been reindexed!")
+        false
+    }
+  }
+
 }
diff --git a/thrall/app/lib/kinesis/MessageTranslator.scala b/thrall/app/lib/kinesis/MessageTranslator.scala
index 0081621c3e..ab3a88d47c 100644
--- a/thrall/app/lib/kinesis/MessageTranslator.scala
+++ b/thrall/app/lib/kinesis/MessageTranslator.scala
@@ -81,6 +81,10 @@ object MessageTranslator extends GridLogging {
         case Some(id) => Right(CreateInstanceMessage(id, updateMessage.lastModified, updateMessage.instance))
         case _ => Left(MissingFieldsException(updateMessage.subject))
       }
+      case ReindexImage => (updateMessage.id) match {
+        case Some(id) => Right(ReindexImageMessage(id, updateMessage.lastModified, updateMessage.instance))
+        case _ => Left(MissingFieldsException(updateMessage.subject))
+      }
       case _ => Left(ProcessorNotFoundException(updateMessage.subject))
     }
   }
diff --git a/thrall/app/lib/kinesis/ThrallEventConsumer.scala b/thrall/app/lib/kinesis/ThrallEventConsumer.scala
index fc4f0ddb9e..988eb2a43e 100644
--- a/thrall/app/lib/kinesis/ThrallEventConsumer.scala
+++ b/thrall/app/lib/kinesis/ThrallEventConsumer.scala
@@ -3,7 +3,7 @@ package lib.kinesis
 import org.apache.pekko.actor.ActorSystem
 import com.gu.mediaservice.GridClient
 import com.gu.mediaservice.lib.auth.Authentication
-import com.gu.mediaservice.lib.aws.UpdateMessage
+import com.gu.mediaservice.lib.aws.{ThrallMessageSender, UpdateMessage}
 import com.gu.mediaservice.lib.events.UsageEvents
 import com.gu.mediaservice.lib.json.{JsonByteArrayUtil, PlayJsonHelpers}
 import com.gu.mediaservice.lib.logging._
@@ -26,7 +26,8 @@ class ThrallEventConsumer(es: ElasticSearch,
   gridClient: GridClient,
   auth: Authentication,
   instanceMessageSender: InstanceMessageSender,
-  usageEvents: UsageEvents
+  usageEvents: UsageEvents,
+  messageSender: ThrallMessageSender
 ) extends PlayJsonHelpers with GridLogging {
 
   private val attemptTimeout = FiniteDuration(20, SECONDS)
@@ -34,7 +35,7 @@ class ThrallEventConsumer(es: ElasticSearch,
   private val attempts = 2
   private val timeout = attemptTimeout * attempts + delay * (attempts - 1)
 
-  private val messageProcessor = new MessageProcessor(es, store, metadataEditorNotifications, gridClient, auth, instanceMessageSender, usageEvents)
+  private val messageProcessor = new MessageProcessor(es, store, metadataEditorNotifications, gridClient, auth, instanceMessageSender, usageEvents, messageSender)
 
   private implicit val implicitActorSystem: ActorSystem = actorSystem
 

From aaeb350816446c0ef3f1763d718da12ffde6675f Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 30 Mar 2025 11:42:15 +0100
Subject: [PATCH 41/45] Reindex should write projection results as
 UpsertFromProjectionMessage like the controller end point.

---
 thrall/app/lib/kinesis/MessageProcessor.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/thrall/app/lib/kinesis/MessageProcessor.scala b/thrall/app/lib/kinesis/MessageProcessor.scala
index f8d0db29c7..490f9ac08b 100644
--- a/thrall/app/lib/kinesis/MessageProcessor.scala
+++ b/thrall/app/lib/kinesis/MessageProcessor.scala
@@ -10,6 +10,7 @@ import com.gu.mediaservice.model.{AddImageLeaseMessage, CreateMigrationIndexMess
 import com.gu.mediaservice.model.usage.{Usage, UsageNotice}
 import com.gu.mediaservice.syntax.MessageSubjects.Image
 import instances.{InstanceMessageSender, InstanceStatusMessage}
+import org.joda.time.DateTime
 // import all except `Right`, which otherwise shadows the type used in `Either`s
 import com.gu.mediaservice.model.{Right => _, _}
 import com.gu.mediaservice.syntax.MessageSubjects
@@ -278,7 +279,7 @@ class MessageProcessor(
     gridClient.getImageLoaderProjection(mediaId, auth.innerServiceCall).map { maybeImage =>
       logger.info(s"Projected ${instance.id} / $mediaId to $maybeImage}")
       maybeImage.exists { image =>
-        val updateMessage = UpdateMessage(subject = Image, image = Some(image), instance = instance)
+        val updateMessage = UpsertFromProjectionMessage(image.id, image, DateTime.now, instance)
         logger.info(s"Publishing projected image as a thrall image message: ${updateMessage.id}")
         messageSender.publish(updateMessage)
         true

From 577844010c21d53ceb36325647267194329058af Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 18 Apr 2025 16:40:18 +0100
Subject: [PATCH 42/45] Reindex logging message.

---
 thrall/app/controllers/ThrallController.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/thrall/app/controllers/ThrallController.scala b/thrall/app/controllers/ThrallController.scala
index bc978b274b..acad7aaf94 100644
--- a/thrall/app/controllers/ThrallController.scala
+++ b/thrall/app/controllers/ThrallController.scala
@@ -326,7 +326,7 @@ class ThrallController(
 
     logger.info(s"Reindex requested for instance ${instance.id}")
     val mediaIds = getMediaIdsFromS3(Seq.empty, None)
-    logger.info(s"Reindexing ${mediaIds.size} images for instance ${instance.id}")
+    logger.info(s"Queuing reindex requests for ${mediaIds.size} images for instance ${instance.id}")
     mediaIds.foreach { mediaId =>
       messageSender.publish(
         UpdateMessage(

From 9e4ffea9cd5c6a46126b375ff183dd4b4a359204 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Fri, 18 Apr 2025 16:53:31 +0100
Subject: [PATCH 43/45] Reindex messages are put into the low priority queue.

---
 thrall/app/ThrallComponents.scala             | 3 ++-
 thrall/app/controllers/ThrallController.scala | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/thrall/app/ThrallComponents.scala b/thrall/app/ThrallComponents.scala
index a6033ab2f7..671d2f9f44 100644
--- a/thrall/app/ThrallComponents.scala
+++ b/thrall/app/ThrallComponents.scala
@@ -63,6 +63,7 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr
   Await.ready(ensureIndexes(), 60 seconds)
 
   val messageSender = new ThrallMessageSender(config.thrallKinesisStreamConfig)
+  val lowPriorityMessageSender = new ThrallMessageSender(config.thrallKinesisLowPriorityStreamConfig)
 
   val highPriorityKinesisConfig: KclPekkoStreamConfig = KinesisConfig.kinesisConfig(config.kinesisConfig)
   val lowPriorityKinesisConfig: KclPekkoStreamConfig = KinesisConfig.kinesisConfig(config.kinesisLowPriorityConfig)
@@ -124,7 +125,7 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr
   val softDeletedMetadataTable = new SoftDeletedMetadataTable(config)
   val maybeCustomReapableEligibility = config.maybeReapableEligibilityClass(applicationLifecycle)
 
-  val thrallController = new ThrallController(es, store, migrationSourceWithSender.send, messageSender, actorSystem, auth, config.services, controllerComponents, gridClient, s3, config.imageBucket)
+  val thrallController = new ThrallController(es, store, migrationSourceWithSender.send, messageSender, actorSystem, auth, config.services, controllerComponents, gridClient, s3, config.imageBucket, lowPriorityMessageSender)
   val reaperController = new ReaperController(es, store, authorisation, config, actorSystem.scheduler, maybeCustomReapableEligibility, softDeletedMetadataTable, thrallMetrics, auth, config.services, controllerComponents, wsClient, events)
   val healthCheckController = new HealthCheck(es, streamRunning.isCompleted, config, controllerComponents)
 
diff --git a/thrall/app/controllers/ThrallController.scala b/thrall/app/controllers/ThrallController.scala
index acad7aaf94..4cb8a34da7 100644
--- a/thrall/app/controllers/ThrallController.scala
+++ b/thrall/app/controllers/ThrallController.scala
@@ -41,6 +41,7 @@ class ThrallController(
   gridClient: GridClient,
   s3: S3,
   imageBucket: S3Bucket,
+  lowPriorityMessageSender: ThrallMessageSender
 )(implicit val ec: ExecutionContext) extends BaseControllerWithLoginRedirects with GridLogging with InstanceForRequest {
 
   private val numberFormatter: Long => String = java.text.NumberFormat.getIntegerInstance().format
@@ -328,7 +329,7 @@ class ThrallController(
     val mediaIds = getMediaIdsFromS3(Seq.empty, None)
     logger.info(s"Queuing reindex requests for ${mediaIds.size} images for instance ${instance.id}")
     mediaIds.foreach { mediaId =>
-      messageSender.publish(
+      lowPriorityMessageSender.publish(
         UpdateMessage(
           subject = ReindexImage,
           id = Some(mediaId),

From 14e5fdfc1f463da0d45f18941370ee5a57adc338 Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sat, 13 Dec 2025 10:48:57 +0000
Subject: [PATCH 44/45] Extract bucket base URL function. Upload bucket
 connectSource is supplied by the bucket object; knows about path style bucket
 URLs.

Preserve the objectUrl is http contract.
Not sure if this is important or not.
---
 .../gu/mediaservice/lib/aws/S3Bucket.scala    | 19 ++++++++++---------
 kahuna/app/lib/KahunaConfig.scala             |  4 ++--
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
index f9ac5b9cc5..bcc3c58ae5 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala
@@ -4,15 +4,8 @@ import java.net.URI
 
 case class S3Bucket(bucket: String, endpoint: String, usesPathStyleURLs: Boolean) {
   def objectUrl(key: String): URI = {
-    val s3Endpoint = endpoint
-    val bucketName = bucket
-    val bucketUrl = if (usesPathStyleURLs) {
-      new URI(s"http://$s3Endpoint/$bucketName/$key")
-    } else {
-      val bucketHost = s"$bucketName.$s3Endpoint"
-      new URI("http", bucketHost, s"/$key", null)
-    }
-    bucketUrl
+    val bucketBaseURL = bucketURL()
+    new URI("http", bucketBaseURL.getHost, bucketBaseURL.getPath + key, null)
   }
 
   def keyFromS3URL(url: URI): String = {
@@ -23,4 +16,12 @@ case class S3Bucket(bucket: String, endpoint: String, usesPathStyleURLs: Boolean
     }
   }
 
+  def bucketURL(): URI = {
+    if (usesPathStyleURLs) {
+      new URI("https", endpoint, s"/$bucket/", null)
+    } else {
+      new URI("https", s"$bucket.$endpoint", "/", null)
+    }
+  }
+
 }
diff --git a/kahuna/app/lib/KahunaConfig.scala b/kahuna/app/lib/KahunaConfig.scala
index 2c8223431d..1ddb1bde36 100644
--- a/kahuna/app/lib/KahunaConfig.scala
+++ b/kahuna/app/lib/KahunaConfig.scala
@@ -2,6 +2,7 @@ package lib
 
 import com.gu.mediaservice.lib.auth.Permissions.Pinboard
 import com.gu.mediaservice.lib.auth.SimplePermission
+import com.gu.mediaservice.lib.aws.S3Bucket
 import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources}
 import com.gu.mediaservice.model.Instance
 import play.api.libs.json._
@@ -50,8 +51,7 @@ class KahunaConfig(resources: GridConfigResources) extends CommonConfig(resource
 
   val frameAncestors: Set[String] = getStringSet("security.frameAncestors")
   val connectSources: Set[String] = getStringSet("security.connectSources") ++ maybeIngestBucket.map { ingestBucket =>
-    if (isDev) "https://localstack.media.local.dev-gutools.co.uk"
-    else s"https://${ingestBucket.bucket}.${ingestBucket.endpoint}"
+    ingestBucket.bucketURL().toURL.toExternalForm
   } ++ telemetryUri
   val fontSources: Set[String] = getStringSet("security.fontSources")
   val imageSources: Set[String] = getStringSet("security.imageSources")

From 4f433ea72710263d046a5a36088506475a3e0fcf Mon Sep 17 00:00:00 2001
From: Tony McCrae <tony@eelpieconsulting.co.uk>
Date: Sun, 25 Jan 2026 12:11:28 +0000
Subject: [PATCH 45/45] Crops bucket can be on non AWS bucket.

---
 cropper/app/lib/CropperConfig.scala | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cropper/app/lib/CropperConfig.scala b/cropper/app/lib/CropperConfig.scala
index 979824eebb..4a4739d8ce 100644
--- a/cropper/app/lib/CropperConfig.scala
+++ b/cropper/app/lib/CropperConfig.scala
@@ -9,9 +9,9 @@ import java.io.File
 
 class CropperConfig(resources: GridConfigResources) extends CommonConfig(resources) {
   val imgPublishingBucket: S3Bucket = S3Bucket(
-    string("publishing.image.bucket"),
-    S3.AmazonAwsS3Endpoint,
-    usesPathStyleURLs = false
+    string("publishing.image.bucket.name"),
+    string("publishing.image.bucket.endpoint"),
+    boolean("publishing.image.bucket.pathStyleURLs")
   )
   val canDownloadCrop: Boolean = boolean("canDownloadCrop")