diff --git a/README.md b/README.md index ca3866d..189629c 100644 --- a/README.md +++ b/README.md @@ -164,4 +164,8 @@ option is selected as target database. ### Diagnostics dashboard -For **ObjectStorage API Server** configuration the following section should be modified: \ No newline at end of file +For **ObjectStorage API Server** configuration the following section should be modified: + +If GCS is selected, please make sure Cloud Resource Manager API is enabled. + +Currently max object size is 1GB, will be changed in the future. \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/repository/common/RepositoryConfigurationHelper.java b/api-server/src/main/java/com/objectstorage/repository/common/RepositoryConfigurationHelper.java index 2b74fb8..b0f0eab 100644 --- a/api-server/src/main/java/com/objectstorage/repository/common/RepositoryConfigurationHelper.java +++ b/api-server/src/main/java/com/objectstorage/repository/common/RepositoryConfigurationHelper.java @@ -7,9 +7,7 @@ import jakarta.enterprise.context.ApplicationScoped; import org.apache.commons.lang3.RandomStringUtils; -import java.util.Arrays; import java.util.List; -import java.util.Optional; import java.util.StringJoiner; /** @@ -24,7 +22,7 @@ public class RepositoryConfigurationHelper { * @return packed external credentials signature. */ private String packExternalCredentials(String ...values) { - StringJoiner result = new StringJoiner(":"); + StringJoiner result = new StringJoiner("|"); for (String value : values) { result.add(value); @@ -40,7 +38,7 @@ private String packExternalCredentials(String ...values) { * @return unpacked external credentials signature. */ private List unpackExternalCredentials(String credentials) { - return List.of(credentials.split(":")); + return List.of(credentials.split("\\|")); } /** diff --git a/api-server/src/main/java/com/objectstorage/resource/ContentResource.java b/api-server/src/main/java/com/objectstorage/resource/ContentResource.java index fc7abf4..f3fe7c4 100644 --- a/api-server/src/main/java/com/objectstorage/resource/ContentResource.java +++ b/api-server/src/main/java/com/objectstorage/resource/ContentResource.java @@ -118,7 +118,12 @@ public byte[] v1ContentBackupDownloadPost(String authorization, ContentBackupDow ValidationSecretsApplication validationSecretsApplication = resourceConfigurationHelper.getJwtDetails(authorization); - return processorService.downloadBackup(contentBackupDownload, validationSecretsApplication); + ValidationSecretsUnit validationSecretsUnit = + resourceConfigurationHelper.getConfiguredProvider( + contentBackupDownload.getProvider(), validationSecretsApplication); + + return processorService.downloadBackup( + contentBackupDownload.getLocation(), validationSecretsUnit, validationSecretsApplication); } /** diff --git a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java index 42bfd85..826cb4a 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java @@ -144,7 +144,9 @@ public void process() throws BackupPeriodRetrievalFailureException { try { workspaceFacade.addBackupFile( workspaceUnitKey, - workspaceFacade.createFileUnitKey(properties.getWorkspaceContentBackupUnit()), + workspaceFacade.createBackupFileUnitKey( + repositoryContentApplicationUnit.getProvider().toString(), + properties.getWorkspaceContentBackupUnit()), folderContentUnits); } catch (FileCreationFailureException e) { StateService.getBackupProcessorGuard().unlock(); diff --git a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java index 00d0f39..de2c987 100644 --- a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java +++ b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java @@ -89,7 +89,8 @@ public ContentRetrievalResult retrieveContent(ValidationSecretsApplication valid List backups; try { - backups = workspaceFacade.getBackupUnits(workspaceUnitKey); + backups = workspaceFacade.getBackupUnits( + workspaceUnitKey, validationSecretsUnit.getProvider().toString()); } catch (FileUnitsRetrievalFailureException e) { throw new ProcessorContentRetrievalFailureException(e.getMessage()); } @@ -262,7 +263,7 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); - String fileUnitKey = workspaceFacade.createFileUnitKey(location); + String fileUnitKey = workspaceFacade.createObjectFileUnitKey(location); for (ValidationSecretsUnit validationSecretsUnit : validationSecretsApplication.getSecrets()) { try { @@ -280,6 +281,50 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp throw new ProcessorContentUploadFailureException(e1.getMessage()); } + + RepositoryContentUnitDto repositoryContentLocationUnitDto; + + try { + repositoryContentLocationUnitDto = repositoryFacade.retrieveContentApplication(validationSecretsUnit); + } catch (ContentApplicationRetrievalFailureException e1) { + try { + repositoryExecutor.rollbackTransaction(); + } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentUploadFailureException(e2.getMessage()); + } + + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentUploadFailureException(e1.getMessage()); + } + + try { + if (!vendorFacade.isBucketPresent( + validationSecretsUnit.getProvider(), + validationSecretsUnit.getCredentials().getExternal(), + VendorConfigurationHelper.createBucketName( + repositoryContentLocationUnitDto.getRoot()))) { + vendorFacade.createBucket( + validationSecretsUnit.getProvider(), + validationSecretsUnit.getCredentials().getExternal(), + VendorConfigurationHelper.createBucketName( + repositoryContentLocationUnitDto.getRoot())); + } + } catch (SecretsConversionException | VendorOperationFailureException e1) { + try { + repositoryExecutor.rollbackTransaction(); + } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentUploadFailureException(e2.getMessage()); + } + + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentUploadFailureException(e1.getMessage()); + } } try { @@ -389,23 +434,27 @@ public byte[] downloadObject( } /** - * Downloads given content backup with the help of the given content backup download application. + * Downloads given content backup with the help of the given content backup download location and + * configured provider. * - * @param contentBackupDownload given content backup download application. + * @param location given content object location. + * @param validationSecretsUnit given content secrets unit. * @param validationSecretsApplication given content secrets application. * @return downloaded content backup. * @throws ProcessorContentDownloadFailureException if content backup download operation fails. */ public byte[] downloadBackup( - ContentBackupDownload contentBackupDownload, + String location, + ValidationSecretsUnit validationSecretsUnit, ValidationSecretsApplication validationSecretsApplication) throws ProcessorContentDownloadFailureException { - logger.info(String.format("Downloading content backup for '%s' location", contentBackupDownload.getLocation())); + logger.info(String.format("Downloading content backup for '%s' location", location)); String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); try { - if (!workspaceFacade.isBackupFilePresent(workspaceUnitKey, contentBackupDownload.getLocation())) { + if (!workspaceFacade.isBackupFilePresent( + workspaceUnitKey, validationSecretsUnit.getProvider().toString(), location)) { throw new ProcessorContentDownloadFailureException( new WorkspaceObjectNotPresentException().getMessage()); } @@ -414,7 +463,8 @@ public byte[] downloadBackup( } try { - return workspaceFacade.getBackupFile(workspaceUnitKey, contentBackupDownload.getLocation()); + return workspaceFacade.getBackupFile( + workspaceUnitKey, validationSecretsUnit.getProvider().toString(), location); } catch (FileUnitRetrievalFailureException e) { throw new ProcessorContentDownloadFailureException(e.getMessage()); } @@ -592,19 +642,49 @@ public void removeAll(ValidationSecretsApplication validationSecretsApplication) throw new ProcessorContentRemovalFailureException(e.getMessage()); } + StateService.getTransactionProcessorGuard().lock(); + + try { + repositoryExecutor.beginTransaction(); + } catch (TransactionInitializationFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e.getMessage()); + } + for (ValidationSecretsUnit validationSecretsUnit : validationSecretsApplication.getSecrets()) { try { repositoryFacade.removeTemporateContentByProviderAndSecret(validationSecretsUnit); - } catch (TemporateContentRemovalFailureException e) { - throw new ProcessorContentRemovalFailureException(e.getMessage()); + } catch (TemporateContentRemovalFailureException e1) { + try { + repositoryExecutor.rollbackTransaction(); + } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e2.getMessage()); + } + + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } RepositoryContentUnitDto repositoryContentLocationUnitDto; try { repositoryContentLocationUnitDto = repositoryFacade.retrieveContentApplication(validationSecretsUnit); - } catch (ContentApplicationRetrievalFailureException e) { - throw new ProcessorContentRemovalFailureException(e.getMessage()); + } catch (ContentApplicationRetrievalFailureException e1) { + try { + repositoryExecutor.rollbackTransaction(); + } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e2.getMessage()); + } + + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } try { @@ -614,9 +694,49 @@ public void removeAll(ValidationSecretsApplication validationSecretsApplication) VendorConfigurationHelper.createBucketName( repositoryContentLocationUnitDto.getRoot()) ); - } catch (SecretsConversionException | VendorOperationFailureException e) { - throw new ProcessorContentRemovalFailureException(e.getMessage()); + } catch (SecretsConversionException | VendorOperationFailureException e1) { + try { + repositoryExecutor.rollbackTransaction(); + } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e2.getMessage()); + } + + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e1.getMessage()); + } + + try { + vendorFacade.removeBucket( + validationSecretsUnit.getProvider(), + validationSecretsUnit.getCredentials().getExternal(), + VendorConfigurationHelper.createBucketName( + repositoryContentLocationUnitDto.getRoot())); + } catch (VendorOperationFailureException | SecretsConversionException e1) { + try { + repositoryExecutor.rollbackTransaction(); + } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e2.getMessage()); + } + + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } } + + try { + repositoryExecutor.commitTransaction(); + } catch (TransactionCommitFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e.getMessage()); + } + + StateService.getTransactionProcessorGuard().unlock(); } } \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java b/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java index 2a5acea..f2416d9 100644 --- a/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java +++ b/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java @@ -3,14 +3,17 @@ import com.google.api.gax.paging.Page; import com.google.auth.Credentials; import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.auth.oauth2.UserCredentials; import com.google.cloud.WriteChannel; import com.google.cloud.resourcemanager.ResourceManager; +import com.google.cloud.resourcemanager.ResourceManagerException; import com.google.cloud.resourcemanager.ResourceManagerOptions; import com.google.cloud.resourcemanager.Project; import com.google.cloud.storage.*; import com.objectstorage.dto.VendorObjectListingDto; import com.objectstorage.exception.GCPCredentialsInitializationFailureException; import com.objectstorage.exception.GCSBucketObjectUploadFailureException; +import com.objectstorage.exception.VendorOperationFailureException; import jakarta.enterprise.context.ApplicationScoped; import java.io.ByteArrayInputStream; @@ -38,6 +41,8 @@ public Credentials getCredentials(String secrets) throws GCPCredentialsInitializ return ServiceAccountCredentials.fromStream(new ByteArrayInputStream(secrets.getBytes())); } catch (IOException e) { throw new GCPCredentialsInitializationFailureException(e.getMessage()); + } catch (ClassCastException e) { + throw new GCPCredentialsInitializationFailureException(); } } @@ -47,18 +52,29 @@ public Credentials getCredentials(String secrets) throws GCPCredentialsInitializ * @param name given name of the GCS bucket. * @param credentials given credentials to be used for client configuration. * @return result of the check. + * @throws VendorOperationFailureException if vendor operation fails. */ public Boolean isGCSBucketPresent( Credentials credentials, - String name) { + String name) throws VendorOperationFailureException { Storage storage = StorageOptions.newBuilder() .setCredentials(credentials) .build() .getService(); - return storage - .get(name) - .exists(); + Bucket bucket; + + try { + bucket = storage.get(name); + } catch (StorageException e) { + throw new VendorOperationFailureException(e.getMessage()); + } + + try { + return Objects.nonNull(bucket) && bucket.exists(); + } catch (StorageException e) { + throw new VendorOperationFailureException(e.getMessage()); + } } /** @@ -66,16 +82,21 @@ public Boolean isGCSBucketPresent( * * @param credentials given credentials to be used for client configuration. * @param name given name of the GCS bucket. + * @throws VendorOperationFailureException if vendor operation fails. */ public void createGCSBucket( Credentials credentials, - String name) { + String name) throws VendorOperationFailureException { Storage storage = StorageOptions.newBuilder() .setCredentials(credentials) .build() .getService(); - storage.create(BucketInfo.newBuilder(name).build()); + try { + storage.create(BucketInfo.newBuilder(name).build()); + } catch (StorageException e) { + throw new VendorOperationFailureException(e.getMessage()); + } } /** @@ -83,16 +104,21 @@ public void createGCSBucket( * * @param credentials given credentials to be used for client configuration. * @param name given name of the GCS bucket. + * @throws VendorOperationFailureException if vendor operation fails. */ public void removeGCSBucket( Credentials credentials, - String name) { + String name) throws VendorOperationFailureException { Storage storage = StorageOptions.newBuilder() .setCredentials(credentials) .build() .getService(); - storage.delete(name); + try { + storage.delete(name); + } catch (StorageException e) { + throw new VendorOperationFailureException(e.getMessage()); + } } /** @@ -130,19 +156,30 @@ public void uploadObjectToGCSBucket( * @param bucketName given name of the GCS bucket. * @param fileName given name of the file to be retrieved. * @return result of the check. + * @throws VendorOperationFailureException if vendor operation fails. */ public Boolean isObjectPresentInBucket( Credentials credentials, String bucketName, - String fileName) { + String fileName) throws VendorOperationFailureException { Storage storage = StorageOptions.newBuilder() .setCredentials(credentials) .build() .getService(); - Blob blob = storage.get(BlobId.of(bucketName, fileName)); + Blob blob; + + try { + blob = storage.get(BlobId.of(bucketName, fileName)); + } catch (StorageException e) { + throw new VendorOperationFailureException(e.getMessage()); + } - return Objects.nonNull(blob) && blob.exists(); + try { + return Objects.nonNull(blob) && blob.exists(); + } catch (StorageException e) { + throw new VendorOperationFailureException(e.getMessage()); + } } /** @@ -152,19 +189,30 @@ public Boolean isObjectPresentInBucket( * @param bucketName given name of the GCS bucket. * @param fileName given name of the file to be retrieved. * @return retrieved object content. + * @throws VendorOperationFailureException if vendor operation fails. */ public byte[] retrieveObjectFromGCSBucket( Credentials credentials, String bucketName, - String fileName) { + String fileName) throws VendorOperationFailureException { Storage storage = StorageOptions.newBuilder() .setCredentials(credentials) .build() .getService(); - Blob blob = storage.get(BlobId.of(bucketName, fileName)); + Blob blob; - return blob.getContent(); + try { + blob = storage.get(BlobId.of(bucketName, fileName)); + } catch (StorageException e) { + throw new VendorOperationFailureException(e.getMessage()); + } + + try { + return blob.getContent(); + } catch (StorageException e) { + throw new VendorOperationFailureException(e.getMessage()); + } } /** @@ -173,22 +221,27 @@ public byte[] retrieveObjectFromGCSBucket( * @param credentials given credentials to be used for client configuration. * @param bucketName given name of the GCS bucket. * @return listed objects. + * @throws VendorOperationFailureException if vendor operation fails. */ public List listObjectsFromGCSBucket( Credentials credentials, - String bucketName) { + String bucketName) throws VendorOperationFailureException { Storage storage = StorageOptions.newBuilder() .setCredentials(credentials) .build() .getService(); - Page blobs = storage.list(bucketName); + try { + Page blobs = storage.list(bucketName); - return StreamSupport.stream(blobs.iterateAll().spliterator(), false) - .map(element -> VendorObjectListingDto.of( - element.getBlobId().getName(), - element.getUpdateTimeOffsetDateTime().toEpochSecond())) - .toList(); + return StreamSupport.stream(blobs.iterateAll().spliterator(), false) + .map(element -> VendorObjectListingDto.of( + element.getBlobId().getName(), + element.getUpdateTimeOffsetDateTime().toEpochSecond())) + .toList(); + } catch (StorageException e) { + throw new VendorOperationFailureException(e.getMessage()); + } } /** @@ -197,17 +250,22 @@ public List listObjectsFromGCSBucket( * @param credentials given credentials to be used for client configuration. * @param bucketName given name of the GCS bucket. * @param fileName given name of the file to be removed. + * @throws VendorOperationFailureException if vendor operation fails. */ public void removeObjectFromGCSBucket( Credentials credentials, String bucketName, - String fileName) { + String fileName) throws VendorOperationFailureException { Storage storage = StorageOptions.newBuilder() .setCredentials(credentials) .build() .getService(); - storage.delete(BlobId.of(bucketName, fileName)); + try { + storage.delete(BlobId.of(bucketName, fileName)); + } catch (StorageException e) { + throw new VendorOperationFailureException(e.getMessage()); + } } /** @@ -215,19 +273,24 @@ public void removeObjectFromGCSBucket( * * @param credentials given credentials to be used for client configuration. * @param bucketName given name of the GCS bucket. + * @throws VendorOperationFailureException if vendor operation fails. */ public void removeAllObjectsFromGCSBucket( Credentials credentials, - String bucketName) { + String bucketName) throws VendorOperationFailureException { Storage storage = StorageOptions.newBuilder() .setCredentials(credentials) .build() .getService(); - Page blobs = storage.list(bucketName); + try { + Page blobs = storage.list(bucketName); - for (Blob blob : blobs.iterateAll()) { - blob.delete(Blob.BlobSourceOption.generationMatch()); + for (Blob blob : blobs.iterateAll()) { + blob.delete(Blob.BlobSourceOption.generationMatch()); + } + } catch (StorageException e) { + throw new VendorOperationFailureException(e.getMessage()); } } @@ -243,8 +306,12 @@ public Boolean isCallerValid(Credentials credentials) { .build() .getService(); - for (Project project : resourceManager.list().iterateAll()) { - return true; + try { + for (Project project : resourceManager.list().iterateAll()) { + return true; + } + } catch (ResourceManagerException e) { + return false; } return false; diff --git a/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java b/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java index d8ea071..2bce15d 100644 --- a/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java +++ b/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java @@ -1,5 +1,6 @@ package com.objectstorage.service.vendor.s3; +import com.amazonaws.SdkClientException; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; @@ -389,7 +390,7 @@ public Boolean isCallerValid(AWSCredentialsProvider awsCredentialsProvider, Stri .build() .getCallerIdentity(new GetCallerIdentityRequest()) .getArn()); - } catch (AWSSecurityTokenServiceException e) { + } catch (SdkClientException e) { return false; } } diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java index 6d90ec5..f42097a 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java @@ -11,7 +11,9 @@ import jakarta.inject.Inject; import java.io.*; +import java.nio.file.Path; import java.time.Instant; +import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; @@ -36,7 +38,9 @@ public class WorkspaceFacade { * @return created workspace unit key. */ public String createWorkspaceUnitKey(ValidationSecretsApplication validationSecretsApplication) { - return validationSecretsApplication.getSecrets().stream().map(element -> + return validationSecretsApplication.getSecrets().stream() + .sorted(Comparator.comparing(element -> element.getProvider().toString())) + .map(element -> switch (element.getProvider()) { case S3 -> workspaceService.createUnitKey( element.getProvider().toString(), @@ -57,13 +61,29 @@ public String createWorkspaceUnitKey(ValidationSecretsApplication validationSecr * @param name given file name. * @return created file unit key. */ - public String createFileUnitKey(String name) { + public String createObjectFileUnitKey(String name) { Instant timestamp = Instant.now(); String fileUnit = workspaceService.createUnitKey(name, Instant.now().toString()); - return String.format("%s-%s-%d", name, fileUnit, timestamp.toEpochMilli()); + return String.format("%s-%d", fileUnit, timestamp.toEpochMilli()); + } + + /** + * Creates backup file unit key with the help of the given file name, selected provider and current datetime. + * + * @param provider given provider name. + * @param name given file name. + * @return created file unit key. + */ + public String createBackupFileUnitKey(String provider, String name) { + Instant timestamp = Instant.now(); + + String fileUnit = + workspaceService.createUnitKey(name, provider, Instant.now().toString()); + + return String.format("%s/%s-%d", provider, fileUnit, timestamp.toEpochMilli()); } /** @@ -154,25 +174,31 @@ public Boolean isObjectFilePresent(String workspaceUnitKey, String name) throws * Checks if backup file with the given name exists in the workspace with the given workspace unit key. * * @param workspaceUnitKey given user workspace unit key. + * @param provider given provider name. * @param name given file name. * @return result of the check. * @throws FileExistenceCheckFailureException if file existence check failed. */ - public Boolean isBackupFilePresent(String workspaceUnitKey, String name) throws FileExistenceCheckFailureException { + public Boolean isBackupFilePresent(String workspaceUnitKey, String provider, String name) + throws FileExistenceCheckFailureException { return workspaceService.isContentFilePresent( - workspaceUnitKey, properties.getWorkspaceContentBackupDirectory(), name); + workspaceUnitKey, properties.getWorkspaceContentBackupDirectory(), Path.of(provider, name).toString()); } /** * Retrieves backup units from the workspace with the given workspace unit key. * * @param workspaceUnitKey given user workspace unit key. + * @param provider given provider name. * @return retrieved backup units. * @throws FileUnitsRetrievalFailureException if file units retrieval fails. */ - public List getBackupUnits(String workspaceUnitKey) throws FileUnitsRetrievalFailureException { + public List getBackupUnits(String workspaceUnitKey, String provider) + throws FileUnitsRetrievalFailureException { return workspaceService - .getContentUnits(workspaceUnitKey, properties.getWorkspaceContentBackupDirectory()) + .getContentUnits( + workspaceUnitKey, + Path.of(properties.getWorkspaceContentBackupDirectory(), provider).toString()) .stream() .map(ContentRetrievalBackupUnit::of) .toList(); @@ -196,12 +222,15 @@ public byte[] getObjectFile(String workspaceUnitKey, String name) throws FileUni * byte array. * * @param workspaceUnitKey given user workspace unit key. + * @param provider given provider name. * @param name given file name. * @return retrieved file as compressed byte array. * @throws FileUnitRetrievalFailureException if file unit retrieval fails. */ - public byte[] getBackupFile(String workspaceUnitKey, String name) throws FileUnitRetrievalFailureException { - return workspaceService.getContentFile(workspaceUnitKey, properties.getWorkspaceContentBackupDirectory(), name); + public byte[] getBackupFile(String workspaceUnitKey, String provider, String name) + throws FileUnitRetrievalFailureException { + return workspaceService.getContentFile( + workspaceUnitKey, properties.getWorkspaceContentBackupDirectory(), Path.of(provider, name).toString()); } /** diff --git a/api-server/src/main/openapi/openapi.yml b/api-server/src/main/openapi/openapi.yml index 8937c2d..f58bb68 100644 --- a/api-server/src/main/openapi/openapi.yml +++ b/api-server/src/main/openapi/openapi.yml @@ -312,9 +312,12 @@ components: ContentBackupDownload: required: - location + - provider properties: location: type: string + provider: + $ref: "#/components/schemas/Provider" ContentCleanup: required: - location diff --git a/api-server/src/main/resources/application.properties b/api-server/src/main/resources/application.properties index 0eb7d0c..fb32b8a 100644 --- a/api-server/src/main/resources/application.properties +++ b/api-server/src/main/resources/application.properties @@ -5,6 +5,8 @@ quarkus.smallrye-health.ui.always-include=true quarkus.swagger-ui.always-include=true quarkus.native.builder-image=graalvm quarkus.banner.path=banner.txt +quarkus.http.limits.max-form-attribute-size=1000M +quarkus.http.limits.max-body-size=1000M # Describes database Quarkus configuration. database.name=objectstorage diff --git a/api-server/src/main/resources/liquibase/postgres/config.yaml b/api-server/src/main/resources/liquibase/postgres/config.yaml index affff68..9b001fd 100644 --- a/api-server/src/main/resources/liquibase/postgres/config.yaml +++ b/api-server/src/main/resources/liquibase/postgres/config.yaml @@ -107,7 +107,6 @@ databaseChangeLog: type: TEXT constraints: nullable: false - unique: true - column: name: created_at type: BIGINT diff --git a/api-server/src/main/resources/liquibase/sqlite3/config.yaml b/api-server/src/main/resources/liquibase/sqlite3/config.yaml index 2443eb8..6119b6c 100644 --- a/api-server/src/main/resources/liquibase/sqlite3/config.yaml +++ b/api-server/src/main/resources/liquibase/sqlite3/config.yaml @@ -107,7 +107,6 @@ databaseChangeLog: type: VARCHAR constraints: nullable: false - unique: true - column: name: created_at type: LONG diff --git a/cli/src/main/java/com/objectstorage/dto/DownloadBackupExternalCommandDto.java b/cli/src/main/java/com/objectstorage/dto/DownloadBackupExternalCommandDto.java index b204e10..5f06651 100644 --- a/cli/src/main/java/com/objectstorage/dto/DownloadBackupExternalCommandDto.java +++ b/cli/src/main/java/com/objectstorage/dto/DownloadBackupExternalCommandDto.java @@ -12,6 +12,8 @@ public class DownloadBackupExternalCommandDto { private ConfigEntity config; + private String provider; + private String outputLocation; private String location; diff --git a/cli/src/main/java/com/objectstorage/service/client/content/upload/object/UploadContentObjectClientService.java b/cli/src/main/java/com/objectstorage/service/client/content/upload/object/UploadContentObjectClientService.java index 15472c4..fa0d8c9 100644 --- a/cli/src/main/java/com/objectstorage/service/client/content/upload/object/UploadContentObjectClientService.java +++ b/cli/src/main/java/com/objectstorage/service/client/content/upload/object/UploadContentObjectClientService.java @@ -47,6 +47,7 @@ public Void process(ContentUploadObjectRequestDto input) throws ApiServerOperati input.getFile()) .block(); } catch (WebClientResponseException e) { + System.out.println(e.getMessage()); throw new ApiServerOperationFailureException(e.getResponseBodyAsString()); } catch (WebClientRequestException e) { throw new ApiServerOperationFailureException(new ApiServerNotAvailableException(e.getMessage()).getMessage()); diff --git a/cli/src/main/java/com/objectstorage/service/command/BaseCommandService.java b/cli/src/main/java/com/objectstorage/service/command/BaseCommandService.java index 0bf73c7..0958d44 100644 --- a/cli/src/main/java/com/objectstorage/service/command/BaseCommandService.java +++ b/cli/src/main/java/com/objectstorage/service/command/BaseCommandService.java @@ -463,6 +463,7 @@ private void downloadObject( * Provides access to download backup command service. * * @param configLocation given custom configuration file location. + * @param provider given selected provider name. * @param outputLocation given output file location. * @param location given backup content backup location name. */ @@ -470,6 +471,7 @@ private void downloadObject( private void downloadBackup( @Option(names = {"--config"}, description = "A location of configuration file", defaultValue = "null") String configLocation, + @Option(names = {"--provider"}, description = "A name of selected provider", required = true) String provider, @Option(names = {"--output"}, description = "A path for the file to be downloaded", required = true) String outputLocation, @Option(names = {"--location"}, description = "A name of backup content location", required = true) String location) { @@ -477,6 +479,12 @@ private void downloadBackup( configLocation = properties.getConfigDefaultLocation(); } + if (!CommandConfigurationHelper.isProviderValid(provider)) { + logger.fatal(new ProviderIsNotValidException().getMessage()); + + return; + } + visualizationState.setLabel(downloadCommandVisualizationLabel); visualizationService.process(); @@ -501,6 +509,7 @@ private void downloadBackup( try { downloadBackupExternalCommandService.process(DownloadBackupExternalCommandDto.of( configService.getConfig(), + provider, outputLocation, location)); } catch (ApiServerOperationFailureException e) { diff --git a/cli/src/main/java/com/objectstorage/service/command/external/download/backup/DownloadBackupExternalCommandService.java b/cli/src/main/java/com/objectstorage/service/command/external/download/backup/DownloadBackupExternalCommandService.java index 800dfe9..373ee6a 100644 --- a/cli/src/main/java/com/objectstorage/service/command/external/download/backup/DownloadBackupExternalCommandService.java +++ b/cli/src/main/java/com/objectstorage/service/command/external/download/backup/DownloadBackupExternalCommandService.java @@ -3,6 +3,7 @@ import com.objectstorage.converter.ConfigCredentialsToContentCredentialsConverter; import com.objectstorage.converter.ConfigProviderToContentProviderConverter; import com.objectstorage.converter.CredentialsConverter; +import com.objectstorage.converter.SelectedProviderToContentProviderConverter; import com.objectstorage.dto.ContentDownloadBackupRequestDto; import com.objectstorage.dto.DownloadBackupExternalCommandDto; import com.objectstorage.dto.ProcessedCredentialsDto; @@ -119,7 +120,9 @@ public void process(DownloadBackupExternalCommandDto downloadBackupExternalComma ContentDownloadBackupRequestDto request = ContentDownloadBackupRequestDto.of( validationSecretsApplicationResult.getToken(), - ContentBackupDownload.of(downloadBackupExternalCommand.getLocation())); + ContentBackupDownload.of(downloadBackupExternalCommand.getLocation(), + SelectedProviderToContentProviderConverter.convert( + downloadBackupExternalCommand.getProvider()))); byte[] contentBackupDownloadResult = downloadContentBackupClientService.process(request); diff --git a/cli/src/main/resources/application.properties b/cli/src/main/resources/application.properties index 9791eb4..c31ff0e 100644 --- a/cli/src/main/resources/application.properties +++ b/cli/src/main/resources/application.properties @@ -1,5 +1,6 @@ # Describes Spring related properties. spring.main.web-application-type=NONE +spring.codec.max-in-memory-size=-1 # Describes timeout used for ObjectStorage API Server client. rest-client.timeout=15 diff --git a/samples/config/client/user.yaml b/samples/config/client/user.yaml index da64089..fdff3f9 100644 --- a/samples/config/client/user.yaml +++ b/samples/config/client/user.yaml @@ -11,11 +11,11 @@ service: # Represents credentials used for the selected provider. credentials: # Represents session identificator, used to distinguish different workspaces and thus separate content inside. - id: "1" + id: "3" # Represents a path to the credentials CSV file, which contains both access # and secret keys. Make sure you don't use "~" symbol. - file: "/Users/yaroslavsvitlytskyi/.aws/credentials/rootkey.csv" + file: "/Users/objectstorage/.aws/credentials/rootkey.csv" # Represents a select region where the deployment of infrastructure will be performed. # Remember that it may influence the availability of the ObjectStorage deployed infrastructure. @@ -23,6 +23,17 @@ service: # This option can be applied to "s3" provider only. region: "us-west-2" + - provider: "gcs" + + # Represents credentials used for the selected provider. + credentials: + # Represents session identificator, used to distinguish different workspaces and thus separate content inside. + id: "1" + + # Represents a path to the credentials CSV file, which contains both access + # and secret keys. Make sure you don't use "~" symbol. + file: "/Users/objectstorage/.config/gcloud/application_default_credentials.json" + # Represents section used for ObjectStorage API Server configuration. api-server: # Represents address for the host of ObjectStorage API Server.