diff --git a/README.md b/README.md index 9b9b22c..559538a 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,6 @@ Features: * Provides configurable data backups per vendor in the same workspace * Makes user interaction similar to filesystem -! A mapping of file system to external vendor providers. - ![](./docs/high-level-design.png) ![](./docs/detailed-design.png) diff --git a/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitsToValidationSecretsApplicationConverter.java b/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitsToValidationSecretsApplicationConverter.java new file mode 100644 index 0000000..180c75b --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitsToValidationSecretsApplicationConverter.java @@ -0,0 +1,32 @@ +package com.objectstorage.converter; + +import com.objectstorage.dto.ContentCompoundUnitDto; +import com.objectstorage.model.ValidationSecretsApplication; +import com.objectstorage.model.ValidationSecretsUnit; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents content compounds units to validation secrets application converter; + */ +public class ContentCompoundUnitsToValidationSecretsApplicationConverter { + + /** + * Converts given content compound units to validation secrets application. + * + * @param contentCompoundUnits given content compounds units. + * @return converted validation secrets application. + */ + public static ValidationSecretsApplication convert(List contentCompoundUnits) { + List validationSecretsUnits = new ArrayList<>(); + + contentCompoundUnits.forEach( + element -> validationSecretsUnits.add( + ValidationSecretsUnit.of( + element.getProvider(), + element.getCredentials()))); + + return ValidationSecretsApplication.of(validationSecretsUnits); + } +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/converter/RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter.java b/api-server/src/main/java/com/objectstorage/converter/RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter.java new file mode 100644 index 0000000..8826ad1 --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/converter/RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter.java @@ -0,0 +1,34 @@ +package com.objectstorage.converter; + +import com.objectstorage.dto.ContentCompoundUnitDto; +import com.objectstorage.dto.RepositoryContentApplicationUnitDto; +import com.objectstorage.model.ValidationSecretsApplication; +import com.objectstorage.model.ValidationSecretsUnit; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents repository content application units to validation secrets application converter; + */ +public class RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter { + + /** + * Converts given content compound units to validation secrets application. + * + * @param repositoryContentApplicationUnits given repository content application units. + * @return converted validation secrets application. + */ + public static ValidationSecretsApplication convert( + List repositoryContentApplicationUnits) { + List validationSecretsUnits = new ArrayList<>(); + + repositoryContentApplicationUnits.forEach( + element -> validationSecretsUnits.add( + ValidationSecretsUnit.of( + element.getProvider(), + element.getCredentials()))); + + return ValidationSecretsApplication.of(validationSecretsUnits); + } +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java new file mode 100644 index 0000000..d4643f6 --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java @@ -0,0 +1,29 @@ +package com.objectstorage.dto; + +import com.objectstorage.model.CredentialsFieldsFull; +import com.objectstorage.model.Provider; +import com.objectstorage.model.ValidationSecretsUnit; +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * Represents content compound unit dto. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class ContentCompoundUnitDto { + /** + * Represents root location for internal file system. + */ + private RepositoryContentUnitDto repositoryContentUnitDto; + + /** + * Represents provider. + */ + private Provider provider; + + /** + * Represents full credentials fields. + */ + private CredentialsFieldsFull credentials; +} diff --git a/api-server/src/main/java/com/objectstorage/dto/EarliestTemporateContentDto.java b/api-server/src/main/java/com/objectstorage/dto/EarliestTemporateContentDto.java index 86ef950..a58e9cb 100644 --- a/api-server/src/main/java/com/objectstorage/dto/EarliestTemporateContentDto.java +++ b/api-server/src/main/java/com/objectstorage/dto/EarliestTemporateContentDto.java @@ -6,6 +6,8 @@ import lombok.AllArgsConstructor; import lombok.Getter; +import java.util.List; + /** * Represents dto used to represent the earliest temporate content. */ @@ -13,9 +15,9 @@ @AllArgsConstructor(staticName = "of") public class EarliestTemporateContentDto { /** - * Represents target provider. + * Represents content compound units. */ - private ValidationSecretsApplication validationSecretsApplication; + private List contentCompoundUnits; /** * Represents file location. diff --git a/api-server/src/main/java/com/objectstorage/dto/FolderContentUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/FolderContentUnitDto.java new file mode 100644 index 0000000..691f727 --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/FolderContentUnitDto.java @@ -0,0 +1,23 @@ +package com.objectstorage.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.io.InputStream; + +/** + * Represents folder content unit. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class FolderContentUnitDto { + /** + * Represents folder entity name. + */ + private String location; + + /** + * Represents folder entity content. + */ + private byte[] content; +} diff --git a/api-server/src/main/java/com/objectstorage/dto/RepositoryContentApplicationUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/RepositoryContentApplicationUnitDto.java new file mode 100644 index 0000000..2750cac --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/RepositoryContentApplicationUnitDto.java @@ -0,0 +1,28 @@ +package com.objectstorage.dto; + +import com.objectstorage.model.CredentialsFieldsFull; +import com.objectstorage.model.Provider; +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * Represents repository content application unit. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class RepositoryContentApplicationUnitDto { + /** + * Represents root location for internal file system. + */ + private String root; + + /** + * Represents provider. + */ + private Provider provider; + + /** + * Represents full credentials fields. + */ + private CredentialsFieldsFull credentials; +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/dto/RepositoryTemporateUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/RepositoryTemporateUnitDto.java deleted file mode 100644 index c000c56..0000000 --- a/api-server/src/main/java/com/objectstorage/dto/RepositoryTemporateUnitDto.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.objectstorage.dto; - -import com.objectstorage.model.CredentialsFieldsFull; -import com.objectstorage.model.Provider; -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * Represents repository temporate unit. - */ -@Getter -@AllArgsConstructor(staticName = "of") -public class RepositoryTemporateUnitDto { - /** - * Represents file location. - */ - private String location; - - /** - * Represents selected service provider. - */ - private Provider provider; -} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/dto/TemporateContentUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/TemporateContentUnitDto.java new file mode 100644 index 0000000..720053e --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/TemporateContentUnitDto.java @@ -0,0 +1,36 @@ +package com.objectstorage.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * Represents repository content unit. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class TemporateContentUnitDto { + /** + * Represents provider. + */ + private Integer provider; + + /** + * Represents se. + */ + private Integer secret; + + /** + * Represents file location. + */ + private String location; + + /** + * Represents file hash. + */ + private String hash; + + /** + * Represents created at timestamp. + */ + private Long createdAt; +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/dto/VendorObjectListingDto.java b/api-server/src/main/java/com/objectstorage/dto/VendorObjectListingDto.java new file mode 100644 index 0000000..25e760c --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/VendorObjectListingDto.java @@ -0,0 +1,21 @@ +package com.objectstorage.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * Represents vendor object listing. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class VendorObjectListingDto { + /** + * Represent location. + */ + private String location; + + /** + * Represents created at timestamp. + */ + private Long createdAt; +} diff --git a/api-server/src/main/java/com/objectstorage/entity/common/ConfigEntity.java b/api-server/src/main/java/com/objectstorage/entity/common/ConfigEntity.java index 942eaa3..361f673 100644 --- a/api-server/src/main/java/com/objectstorage/entity/common/ConfigEntity.java +++ b/api-server/src/main/java/com/objectstorage/entity/common/ConfigEntity.java @@ -56,34 +56,6 @@ public static class Security { */ @Getter public static class TemporateStorage { - /** - * Represents all supported content formats, which can be used by ObjectStorage - * Temporate Storage. - */ - @Getter - public enum Format { - @JsonProperty("zip") - ZIP("zip"), - - @JsonProperty("tar") - TAR("tar"); - - private final String value; - - Format(String value) { - this.value = value; - } - - public String toString() { - return value; - } - } - - @Valid - @NotNull - @JsonProperty("format") - public Format format; - @NotNull @JsonProperty("frequency") public String frequency; diff --git a/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java b/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java index 4cf79a1..ec3e7a9 100644 --- a/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java +++ b/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java @@ -68,6 +68,9 @@ public class PropertiesEntity { @ConfigProperty(name = "workspace.content.backup.directory") String workspaceContentBackupDirectory; + @ConfigProperty(name = "workspace.content.backup.unit") + String workspaceContentBackupUnit; + @ConfigProperty(name = "workspace.compression.file.name") String workspaceCompressionFileName; diff --git a/api-server/src/main/java/com/objectstorage/exception/FileUnitsLocationsRetrievalFailureException.java b/api-server/src/main/java/com/objectstorage/exception/FileUnitsLocationsRetrievalFailureException.java deleted file mode 100644 index 310dc1c..0000000 --- a/api-server/src/main/java/com/objectstorage/exception/FileUnitsLocationsRetrievalFailureException.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.objectstorage.exception; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Formatter; - -/** - * Represents exception used when file units locations retrieval operation fails. - */ -public class FileUnitsLocationsRetrievalFailureException extends IOException { - public FileUnitsLocationsRetrievalFailureException() { - this(""); - } - - public FileUnitsLocationsRetrievalFailureException(Object... message) { - super( - new Formatter() - .format("File units locations retrieval operation failed: %s", Arrays.stream(message).toArray()) - .toString()); - } -} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/exception/FileUnitsRetrievalFailureException.java b/api-server/src/main/java/com/objectstorage/exception/FileUnitsRetrievalFailureException.java new file mode 100644 index 0000000..ba8de4e --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/exception/FileUnitsRetrievalFailureException.java @@ -0,0 +1,21 @@ +package com.objectstorage.exception; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Formatter; + +/** + * Represents exception used when file units retrieval operation fails. + */ +public class FileUnitsRetrievalFailureException extends IOException { + public FileUnitsRetrievalFailureException() { + this(""); + } + + public FileUnitsRetrievalFailureException(Object... message) { + super( + new Formatter() + .format("File units retrieval operation failed: %s", Arrays.stream(message).toArray()) + .toString()); + } +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java b/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java index 6fc91d7..32412d8 100644 --- a/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java +++ b/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java @@ -60,7 +60,7 @@ public void insert(Integer provider, Integer secret, String root) throws Reposit * @return retrieved content entities. * @throws RepositoryOperationFailureException if repository operation fails. */ - public List findByProviderAndSecret(Integer provider, Integer secret) throws + public ContentEntity findByProviderAndSecret(Integer provider, Integer secret) throws RepositoryOperationFailureException { ResultSet resultSet; @@ -77,18 +77,12 @@ public List findByProviderAndSecret(Integer provider, Integer sec throw new RepositoryOperationFailureException(e.getMessage()); } - List result = new ArrayList<>(); - Integer id; String root; try { - while (resultSet.next()) { - id = resultSet.getInt("id"); - root = resultSet.getString("root"); - - result.add(ContentEntity.of(id, provider, secret, root)); - } + id = resultSet.getInt("id"); + root = resultSet.getString("root"); } catch (SQLException e1) { try { resultSet.close(); @@ -105,6 +99,55 @@ public List findByProviderAndSecret(Integer provider, Integer sec throw new RepositoryOperationFailureException(e.getMessage()); } + return ContentEntity.of(id, provider, secret, root); + } + + /** + * Retrieves all the persisted content entities. + * + * @return retrieved content entities. + * @throws RepositoryOperationFailureException if repository operation fails. + */ + public List findAll() throws RepositoryOperationFailureException { + ResultSet resultSet; + + try { + resultSet = + repositoryExecutor.performQueryWithResult( + String.format( + "SELECT t.id, t.root, t.provider, t.secret FROM %s as t", + properties.getDatabaseContentTableName())); + + } catch (QueryExecutionFailureException | QueryEmptyResultException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + List result = new ArrayList<>(); + + Integer id; + String root; + Integer provider; + Integer secret; + + try { + while (resultSet.next()) { + id = resultSet.getInt("id"); + root = resultSet.getString("root"); + provider = resultSet.getInt("provider"); + secret = resultSet.getInt("secret"); + + result.add(ContentEntity.of(id, provider, secret, root)); + } + } catch (SQLException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + try { + resultSet.close(); + } catch (SQLException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + return result; } diff --git a/api-server/src/main/java/com/objectstorage/repository/TemporateRepository.java b/api-server/src/main/java/com/objectstorage/repository/TemporateRepository.java index 2030840..5069417 100644 --- a/api-server/src/main/java/com/objectstorage/repository/TemporateRepository.java +++ b/api-server/src/main/java/com/objectstorage/repository/TemporateRepository.java @@ -156,6 +156,7 @@ public TemporateEntity findEarliest() throws RepositoryOperationFailureException /** * Retrieves all the persisted temporate entities with the given hash. * + * @param hash given hash. * @return retrieved temporate entities. * @throws RepositoryOperationFailureException if repository operation fails. */ @@ -212,9 +213,66 @@ public List findByHash(String hash) throws return result; } + /** + * Retrieves all the persisted temporate entities with the given location, provider and secret. + * + * @param location given location. + * @param provider given provider. + * @param secret given secret. + * @return retrieved temporate entity. + * @throws RepositoryOperationFailureException if repository operation fails. + */ + public TemporateEntity findEarliestByLocationProviderAndSecret( + String location, Integer provider, Integer secret) throws RepositoryOperationFailureException { + ResultSet resultSet; + + try { + resultSet = + repositoryExecutor.performQueryWithResult( + String.format( + "SELECT t.id, t.hash, t.created_at FROM %s as t WHERE t.location = '%s' AND t.provider = %d AND t.secret = %d ORDER BY created_at DESC LIMIT 1", + properties.getDatabaseTemporateTableName(), + location, + provider, + secret)); + + } catch (QueryExecutionFailureException | QueryEmptyResultException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + Integer id; + String hash; + Long createdAt; + + try { + id = resultSet.getInt("id"); + hash = resultSet.getString("hash"); + createdAt = resultSet.getLong("created_at"); + + } catch (SQLException e1) { + try { + resultSet.close(); + } catch (SQLException e2) { + throw new RepositoryOperationFailureException(e2.getMessage()); + } + + throw new RepositoryOperationFailureException(e1.getMessage()); + } + + try { + resultSet.close(); + } catch (SQLException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + return TemporateEntity.of(id, provider, secret, location, hash, createdAt); + } + /** * Retrieves all the persisted temporate entities with the given provider and secret. * + * @param provider given provider. + * @param secret given secret. * @return retrieved temporate entities. * @throws RepositoryOperationFailureException if repository operation fails. */ diff --git a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java index e110177..370a41e 100644 --- a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java +++ b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java @@ -1,7 +1,7 @@ package com.objectstorage.repository.facade; -import com.objectstorage.dto.RepositoryContentUnitDto; -import com.objectstorage.dto.EarliestTemporateContentDto; +import com.objectstorage.dto.*; +import com.objectstorage.entity.repository.ContentEntity; import com.objectstorage.entity.repository.ProviderEntity; import com.objectstorage.entity.repository.SecretEntity; import com.objectstorage.entity.repository.TemporateEntity; @@ -9,6 +9,7 @@ import com.objectstorage.model.*; import com.objectstorage.repository.*; import com.objectstorage.repository.common.RepositoryConfigurationHelper; +import com.objectstorage.service.telemetry.TelemetryService; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -41,6 +42,9 @@ public class RepositoryFacade { @Inject SecretRepository secretRepository; + @Inject + TelemetryService telemetryService; + /** * Retrieves filtered content from temporate repository. * @@ -80,7 +84,8 @@ public List retrieveFilteredTemporateContent( } return temporateContent.stream().map( - element -> ContentRetrievalProviderUnit.of(element.getLocation())).toList(); + element -> ContentRetrievalProviderUnit.of( + element.getLocation(), element.getCreatedAt())).toList(); } /** @@ -90,11 +95,17 @@ public List retrieveFilteredTemporateContent( * @throws TemporateContentRetrievalFailureException if temporate content amount retrieval fails. */ public Boolean isTemporateContentPresent() throws TemporateContentRetrievalFailureException { + Integer amount; + try { - return temporateRepository.count() > 0; + amount = temporateRepository.count(); } catch (RepositoryOperationFailureException e) { throw new TemporateContentRetrievalFailureException(e.getMessage()); } + + telemetryService.setTemporateStorageFilesAmount(amount); + + return amount > 0; } /** @@ -124,7 +135,7 @@ public EarliestTemporateContentDto retrieveEarliestTemporateContent() throws Tem throw new TemporateContentRetrievalFailureException(e.getMessage()); } - List validationSecretsUnits = new ArrayList<>(); + List contentCompoundUnits = new ArrayList<>(); for (TemporateEntity temporate : temporateEntities) { ProviderEntity rawProvider; @@ -141,27 +152,98 @@ public EarliestTemporateContentDto retrieveEarliestTemporateContent() throws Tem SecretEntity secret; try { - secret = secretRepository.findById(temporate.getId()); + secret = secretRepository.findById(temporate.getSecret()); } catch (RepositoryOperationFailureException e) { throw new TemporateContentRetrievalFailureException(e.getMessage()); } - CredentialsFieldsFull secrets = + CredentialsFieldsFull credentials = repositoryConfigurationHelper.convertRawSecretsToContentCredentials( provider, secret.getSession(), secret.getCredentials()); - validationSecretsUnits.add(ValidationSecretsUnit.of(provider, secrets)); + ContentEntity contentEntity; + + try { + contentEntity = contentRepository.findByProviderAndSecret(rawProvider.getId(), secret.getId()); + } catch (RepositoryOperationFailureException e) { + throw new TemporateContentRetrievalFailureException(e.getMessage()); + } + + contentCompoundUnits.add( + ContentCompoundUnitDto.of( + RepositoryContentUnitDto.of( + contentEntity.getRoot()), + provider, + credentials)); } return EarliestTemporateContentDto.of( - ValidationSecretsApplication.of(validationSecretsUnits), + contentCompoundUnits, temporateEntity.getLocation(), temporateEntity.getHash(), temporateEntity.getCreatedAt()); } + /** + * Retrieves temporate content from the temporate repository with the given location, provider and secret. + * + * @param location given temporate content location. + * @param validationSecretsUnit given validation secrets unit. + * @return retrieved temporate content. + */ + public TemporateContentUnitDto retrieveTemporateContentByLocationProviderAndSecret(String location, ValidationSecretsUnit validationSecretsUnit) + throws TemporateContentRemovalFailureException { + ProviderEntity provider; + + try { + provider = providerRepository.findByName(validationSecretsUnit.getProvider().toString()); + } catch (RepositoryOperationFailureException e) { + throw new TemporateContentRemovalFailureException(e.getMessage()); + } + + String signature = repositoryConfigurationHelper.getExternalCredentials( + validationSecretsUnit.getProvider(), + validationSecretsUnit.getCredentials().getExternal()); + + try { + if (!secretRepository.isPresentBySessionAndCredentials( + validationSecretsUnit.getCredentials().getInternal().getId(), signature)) { + throw new TemporateContentRemovalFailureException( + new RepositoryContentApplicationNotExistsException().getMessage()); + } + } catch (RepositoryOperationFailureException e) { + throw new TemporateContentRemovalFailureException(e.getMessage()); + } + + SecretEntity secret; + + try { + secret = secretRepository.findBySessionAndCredentials( + validationSecretsUnit.getCredentials().getInternal().getId(), + signature); + } catch (RepositoryOperationFailureException e) { + throw new TemporateContentRemovalFailureException(e.getMessage()); + } + + TemporateEntity temporate; + + try { + temporate = temporateRepository + .findEarliestByLocationProviderAndSecret(location, provider.getId(), secret.getId()); + } catch (RepositoryOperationFailureException ignored) { + return null; + } + + return TemporateContentUnitDto.of( + temporate.getProvider(), + temporate.getSecret(), + temporate.getLocation(), + temporate.getHash(), + temporate.getCreatedAt()); + } + /** * Retrieves content application from the content repository. * @@ -203,15 +285,68 @@ public RepositoryContentUnitDto retrieveContentApplication(ValidationSecretsUnit throw new ContentApplicationRetrievalFailureException(e.getMessage()); } + ContentEntity contentEntity; + + try { + contentEntity = contentRepository + .findByProviderAndSecret(provider.getId(), secret.getId()); + } catch (RepositoryOperationFailureException e) { + throw new ContentApplicationRetrievalFailureException(e.getMessage()); + } + + return RepositoryContentUnitDto.of(contentEntity.getRoot()); + } + + /** + * Retrieves all content applications from the content repository. + * + * @return retrieved all content applications. + * @throws ContentApplicationRetrievalFailureException if content applications retrieval fails. + */ + public List retrieveAllContentApplications() + throws ContentApplicationRetrievalFailureException { + List contentEntities; + try { - return contentRepository - .findByProviderAndSecret(provider.getId(), secret.getId()) - .stream() - .map(element -> RepositoryContentUnitDto.of(element.getRoot())) - .toList().getFirst(); + contentEntities = contentRepository.findAll(); } catch (RepositoryOperationFailureException e) { throw new ContentApplicationRetrievalFailureException(e.getMessage()); } + + List repositoryContentApplicationUnits = new ArrayList<>(); + + for (ContentEntity content : contentEntities) { + ProviderEntity rawProvider; + + try { + rawProvider = providerRepository.findById(content.getProvider()); + } catch (RepositoryOperationFailureException e) { + throw new ContentApplicationRetrievalFailureException(e.getMessage()); + } + + Provider provider = + repositoryConfigurationHelper.convertRawProviderToContentProvider(rawProvider.getName()); + + SecretEntity secret; + + try { + secret = secretRepository.findById(content.getSecret()); + } catch (RepositoryOperationFailureException e) { + throw new ContentApplicationRetrievalFailureException(e.getMessage()); + } + + CredentialsFieldsFull credentials = + repositoryConfigurationHelper.convertRawSecretsToContentCredentials( + provider, + secret.getSession(), + secret.getCredentials()); + + repositoryContentApplicationUnits.add( + RepositoryContentApplicationUnitDto.of( + content.getRoot(), provider, credentials)); + } + + return repositoryContentApplicationUnits; } /** diff --git a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java index 7835a05..453fc2a 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java @@ -1,17 +1,31 @@ package com.objectstorage.service.integration.backup; import com.objectstorage.converter.CronExpressionConverter; -import com.objectstorage.exception.BackupPeriodRetrievalFailureException; -import com.objectstorage.exception.CronExpressionException; +import com.objectstorage.converter.RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter; +import com.objectstorage.dto.FolderContentUnitDto; +import com.objectstorage.dto.RepositoryContentApplicationUnitDto; +import com.objectstorage.entity.common.PropertiesEntity; +import com.objectstorage.exception.*; +import com.objectstorage.model.ContentRetrievalProviderUnit; +import com.objectstorage.model.ValidationSecretsApplication; +import com.objectstorage.repository.facade.RepositoryFacade; import com.objectstorage.service.state.StateService; +import com.objectstorage.service.telemetry.TelemetryService; +import com.objectstorage.service.vendor.VendorFacade; +import com.objectstorage.service.vendor.common.VendorConfigurationHelper; import com.objectstorage.service.workspace.facade.WorkspaceFacade; +import io.quarkus.runtime.Startup; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import com.objectstorage.service.config.ConfigService; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -19,14 +33,29 @@ /** * Provides backup configuration, which will create a local backup of uploaded files. */ +@Startup(value = 900) @ApplicationScoped public class BackupService { + private static final Logger logger = LogManager.getLogger(BackupService.class); + + @Inject + PropertiesEntity properties; + @Inject ConfigService configService; @Inject WorkspaceFacade workspaceFacade; + @Inject + VendorFacade vendorFacade; + + @Inject + RepositoryFacade repositoryFacade; + + @Inject + TelemetryService telemetryService; + private final ScheduledExecutorService scheduledOperationExecutorService = Executors.newSingleThreadScheduledExecutor(); @@ -49,7 +78,85 @@ public void process() throws BackupPeriodRetrievalFailureException { scheduledOperationExecutorService.scheduleWithFixedDelay(() -> { StateService.getBackupProcessorGuard().lock(); + List repositoryContentApplicationUnits; + + try { + repositoryContentApplicationUnits = repositoryFacade.retrieveAllContentApplications(); + } catch (ContentApplicationRetrievalFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + + ValidationSecretsApplication validationSecretsApplication = + RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter + .convert(repositoryContentApplicationUnits); + + String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); + + for (RepositoryContentApplicationUnitDto repositoryContentApplicationUnit : + repositoryContentApplicationUnits) { + List contentRetrievalProviderUnits; + + try { + contentRetrievalProviderUnits = vendorFacade.listAllObjectsFromBucket( + repositoryContentApplicationUnit.getProvider(), + repositoryContentApplicationUnit.getCredentials().getExternal(), + VendorConfigurationHelper.createBucketName( + repositoryContentApplicationUnit.getRoot())); + } catch (SecretsConversionException | BucketObjectRetrievalFailureException | VendorOperationFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + + List folderContentUnits = new ArrayList<>(); + + for (ContentRetrievalProviderUnit contentRetrievalProviderUnit : contentRetrievalProviderUnits) { + byte[] content; + + try { + content = vendorFacade.retrieveObjectFromBucket( + repositoryContentApplicationUnit.getProvider(), + repositoryContentApplicationUnit.getCredentials().getExternal(), + VendorConfigurationHelper.createBucketName( + repositoryContentApplicationUnit.getRoot()), + contentRetrievalProviderUnit.getLocation() + ); + } catch ( + SecretsConversionException | + BucketObjectRetrievalFailureException | + VendorOperationFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + + folderContentUnits.add(FolderContentUnitDto.of( + contentRetrievalProviderUnit.getLocation(), content)); + } + + try { + workspaceFacade.addBackupFile( + workspaceUnitKey, + workspaceFacade.createFileUnitKey(properties.getWorkspaceContentBackupUnit()), + folderContentUnits); + } catch (FileCreationFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + telemetryService.increaseCurrentBackupsAmount(); + } StateService.getBackupProcessorGuard().unlock(); }, 0, period, TimeUnit.MILLISECONDS); diff --git a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java index bd63588..30251c4 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java @@ -1,14 +1,19 @@ package com.objectstorage.service.integration.temporatestorage; +import com.objectstorage.converter.ContentCompoundUnitsToValidationSecretsApplicationConverter; import com.objectstorage.converter.CronExpressionConverter; +import com.objectstorage.dto.ContentCompoundUnitDto; import com.objectstorage.dto.EarliestTemporateContentDto; import com.objectstorage.exception.*; -import com.objectstorage.model.ValidationSecretsUnit; +import com.objectstorage.model.ValidationSecretsApplication; import com.objectstorage.repository.executor.RepositoryExecutor; import com.objectstorage.repository.facade.RepositoryFacade; import com.objectstorage.service.config.ConfigService; import com.objectstorage.service.state.StateService; +import com.objectstorage.service.telemetry.TelemetryService; +import com.objectstorage.service.telemetry.binding.TelemetryBinding; import com.objectstorage.service.vendor.VendorFacade; +import com.objectstorage.service.vendor.common.VendorConfigurationHelper; import com.objectstorage.service.workspace.facade.WorkspaceFacade; import io.quarkus.runtime.Startup; import jakarta.annotation.PostConstruct; @@ -46,6 +51,12 @@ public class TemporateStorageService { @Inject VendorFacade vendorFacade; + @Inject + TelemetryBinding telemetryBinding; + + @Inject + TelemetryService telemetryService; + private final ScheduledExecutorService scheduledOperationExecutorService = Executors.newSingleThreadScheduledExecutor(); @@ -65,6 +76,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { throw new TemporateStoragePeriodRetrievalFailureException(e.getMessage()); } + telemetryBinding.getConfiguredTemporateStorageAwaitTime().set(period); + scheduledOperationExecutorService.scheduleWithFixedDelay(() -> { StateService.getTemporateStorageProcessorGuard().lock(); @@ -77,9 +90,11 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { } catch (TemporateContentRetrievalFailureException e1) { StateService.getTemporateStorageProcessorGuard().unlock(); - logger.fatal(e1.getMessage()); + telemetryService.increaseCloudServiceUploadRetries(); + + logger.error(e1.getMessage()); - throw new RuntimeException(e1); + return; } EarliestTemporateContentDto temporateContentDto; @@ -89,85 +104,120 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { } catch (TemporateContentRetrievalFailureException e) { StateService.getTemporateStorageProcessorGuard().unlock(); - logger.fatal(e.getMessage()); + telemetryService.increaseCloudServiceUploadRetries(); - throw new RuntimeException(e); + logger.error(e.getMessage()); + + return; } + StateService.getTransactionProcessorGuard().lock(); + try { repositoryExecutor.beginTransaction(); } catch (TransactionInitializationFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); - logger.fatal(e.getMessage()); + telemetryService.increaseCloudServiceUploadRetries(); - throw new RuntimeException(e); + logger.error(e.getMessage()); + + return; } try { repositoryFacade.removeTemporateContentByHash(temporateContentDto.getHash()); } catch (TemporateContentRemovalFailureException e1) { - StateService.getTemporateStorageProcessorGuard().unlock(); + telemetryService.increaseCloudServiceUploadRetries(); try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { - logger.fatal(e2.getMessage()); + StateService.getTransactionProcessorGuard().unlock(); - throw new RuntimeException(e2); + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e2.getMessage()); + + return; } - logger.fatal(e1.getMessage()); + StateService.getTransactionProcessorGuard().unlock(); + + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e1.getMessage()); - throw new RuntimeException(e1); + return; } - String workspaceUnitKey = - workspaceFacade.createWorkspaceUnitKey(temporateContentDto.getValidationSecretsApplication()); + ValidationSecretsApplication validationSecretsApplication = + ContentCompoundUnitsToValidationSecretsApplicationConverter.convert( + temporateContentDto.getContentCompoundUnits()); + + String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); byte[] content; try { content = workspaceFacade.getObjectFile(workspaceUnitKey, temporateContentDto.getHash()); } catch (FileUnitRetrievalFailureException e1) { - StateService.getTemporateStorageProcessorGuard().unlock(); + telemetryService.increaseCloudServiceUploadRetries(); try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { - logger.fatal(e2.getMessage()); + StateService.getTransactionProcessorGuard().unlock(); + + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e2.getMessage()); - throw new RuntimeException(e2); + return; } - logger.fatal(e1.getMessage()); + StateService.getTransactionProcessorGuard().unlock(); + + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e1.getMessage()); - throw new RuntimeException(e1); + return; } - for (ValidationSecretsUnit validationSecretsUnit : temporateContentDto.getValidationSecretsApplication() - .getSecrets()) { + for (ContentCompoundUnitDto contentCompoundUnit : temporateContentDto.getContentCompoundUnits()) { try { vendorFacade.uploadObjectToBucket( - validationSecretsUnit.getProvider(), - validationSecretsUnit.getCredentials().getExternal(), - "", + contentCompoundUnit.getProvider(), + contentCompoundUnit.getCredentials().getExternal(), + VendorConfigurationHelper.createBucketName( + contentCompoundUnit.getRepositoryContentUnitDto().getRoot()), temporateContentDto.getLocation(), new ByteArrayInputStream(content)); } catch ( SecretsConversionException | VendorOperationFailureException | BucketObjectUploadFailureException e1) { - StateService.getTemporateStorageProcessorGuard().unlock(); + telemetryService.increaseCloudServiceUploadRetries(); try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { - logger.fatal(e2.getMessage()); + StateService.getTransactionProcessorGuard().unlock(); + + StateService.getTemporateStorageProcessorGuard().unlock(); - throw new RuntimeException(e2); + logger.error(e2.getMessage()); + + return; } + StateService.getTransactionProcessorGuard().unlock(); + + StateService.getTemporateStorageProcessorGuard().unlock(); + logger.info(e1.getMessage()); return; @@ -177,29 +227,47 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { try { workspaceFacade.removeObjectFile(workspaceUnitKey, temporateContentDto.getHash()); } catch (FileRemovalFailureException e1) { - StateService.getTemporateStorageProcessorGuard().unlock(); + telemetryService.increaseCloudServiceUploadRetries(); try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { - logger.fatal(e2.getMessage()); + StateService.getTransactionProcessorGuard().unlock(); - throw new RuntimeException(e2); + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e2.getMessage()); + + return; } - logger.info(e1.getMessage()); + StateService.getTransactionProcessorGuard().unlock(); + + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e1.getMessage()); + + return; } try { repositoryExecutor.commitTransaction(); } catch (TransactionCommitFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); - logger.fatal(e.getMessage()); + telemetryService.increaseCloudServiceUploadRetries(); + + logger.error(e.getMessage()); - throw new RuntimeException(e); + return; } + StateService.getTransactionProcessorGuard().unlock(); + + telemetryService.increaseCurrentCloudServiceUploads(); + StateService.getTemporateStorageProcessorGuard().unlock(); }, 0, period, TimeUnit.MILLISECONDS); } diff --git a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java index f610b44..00d0f39 100644 --- a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java +++ b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java @@ -1,10 +1,12 @@ package com.objectstorage.service.processor; import com.objectstorage.dto.RepositoryContentUnitDto; +import com.objectstorage.dto.TemporateContentUnitDto; import com.objectstorage.exception.*; import com.objectstorage.model.*; import com.objectstorage.repository.executor.RepositoryExecutor; import com.objectstorage.repository.facade.RepositoryFacade; +import com.objectstorage.service.state.StateService; import com.objectstorage.service.telemetry.TelemetryService; import com.objectstorage.service.vendor.VendorFacade; import com.objectstorage.service.vendor.common.VendorConfigurationHelper; @@ -14,9 +16,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * Provides high-level access to ObjectStorage processor operations. @@ -51,6 +55,8 @@ public ContentRetrievalResult retrieveContent(ValidationSecretsApplication valid throws ProcessorContentRetrievalFailureException { List compounds = new ArrayList<>(); + String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); + for (ValidationSecretsUnit validationSecretsUnit : validationSecretsApplication.getSecrets()) { RepositoryContentUnitDto repositoryContentLocationUnitDto; @@ -80,11 +86,19 @@ public ContentRetrievalResult retrieveContent(ValidationSecretsApplication valid VendorOperationFailureException ignored) { } + List backups; + + try { + backups = workspaceFacade.getBackupUnits(workspaceUnitKey); + } catch (FileUnitsRetrievalFailureException e) { + throw new ProcessorContentRetrievalFailureException(e.getMessage()); + } + compounds.add( ContentRetrievalCompound.of( repositoryContentLocationUnitDto.getRoot(), validationSecretsUnit.getProvider().toString(), - List.of(ContentRetrievalUnits.of(pending, uploaded)))); + List.of(ContentRetrievalUnits.of(pending, uploaded, backups)))); } return ContentRetrievalResult.of(compounds); @@ -99,9 +113,13 @@ public ContentRetrievalResult retrieveContent(ValidationSecretsApplication valid */ public void apply(ContentApplication contentApplication, ValidationSecretsApplication validationSecretsApplication) throws ProcessorContentApplicationFailureException { + StateService.getTransactionProcessorGuard().lock(); + try { repositoryExecutor.beginTransaction(); } catch (TransactionInitializationFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e.getMessage()); } @@ -112,9 +130,13 @@ public void apply(ContentApplication contentApplication, ValidationSecretsApplic try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e1.getMessage()); } @@ -134,9 +156,13 @@ public void apply(ContentApplication contentApplication, ValidationSecretsApplic try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e1.getMessage()); } } @@ -144,8 +170,12 @@ public void apply(ContentApplication contentApplication, ValidationSecretsApplic try { repositoryExecutor.commitTransaction(); } catch (TransactionCommitFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e.getMessage()); } + + StateService.getTransactionProcessorGuard().unlock(); } /** @@ -157,58 +187,30 @@ public void apply(ContentApplication contentApplication, ValidationSecretsApplic */ public void withdraw(ValidationSecretsApplication validationSecretsApplication) throws ProcessorContentWithdrawalFailureException { + StateService.getTransactionProcessorGuard().lock(); + try { repositoryExecutor.beginTransaction(); } catch (TransactionInitializationFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentWithdrawalFailureException(e.getMessage()); } for (ValidationSecretsUnit validationSecretsUnit : validationSecretsApplication.getSecrets()) { - RepositoryContentUnitDto repositoryContentLocationUnitDto; - - try { - repositoryContentLocationUnitDto = repositoryFacade.retrieveContentApplication(validationSecretsUnit); - } catch (ContentApplicationRetrievalFailureException e1) { - try { - repositoryExecutor.rollbackTransaction(); - } catch (TransactionRollbackFailureException e2) { - throw new ProcessorContentWithdrawalFailureException(e2.getMessage()); - } - - throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); - } - try { repositoryFacade.withdraw(validationSecretsUnit); } catch (RepositoryContentDestructionFailureException e1) { try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { - throw new ProcessorContentWithdrawalFailureException(e2.getMessage()); - } - - throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); - } + StateService.getTransactionProcessorGuard().unlock(); - try { - if (vendorFacade.isBucketPresent( - validationSecretsUnit.getProvider(), - validationSecretsUnit.getCredentials().getExternal(), - VendorConfigurationHelper.createBucketName( - repositoryContentLocationUnitDto.getRoot()))) { - vendorFacade.removeBucket( - validationSecretsUnit.getProvider(), - validationSecretsUnit.getCredentials().getExternal(), - VendorConfigurationHelper.createBucketName( - repositoryContentLocationUnitDto.getRoot())); - } - } catch (SecretsConversionException | VendorOperationFailureException e1) { - try { - repositoryExecutor.rollbackTransaction(); - } catch (TransactionRollbackFailureException e2) { throw new ProcessorContentWithdrawalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); } } @@ -216,8 +218,12 @@ public void withdraw(ValidationSecretsApplication validationSecretsApplication) try { repositoryExecutor.commitTransaction(); } catch (TransactionCommitFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentWithdrawalFailureException(e.getMessage()); } + + StateService.getTransactionProcessorGuard().unlock(); } /** @@ -233,9 +239,23 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp throws ProcessorContentUploadFailureException { logger.info(String.format("Uploading content at '%s' location", location)); + StateService.getTransactionProcessorGuard().lock(); + try { repositoryExecutor.beginTransaction(); } catch (TransactionInitializationFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentUploadFailureException(e.getMessage()); + } + + Integer fileSize; + + try { + fileSize = file.available(); + } catch (IOException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentUploadFailureException(e.getMessage()); } @@ -251,9 +271,13 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentUploadFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentUploadFailureException(e1.getMessage()); } } @@ -264,8 +288,11 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentUploadFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); throw new ProcessorContentUploadFailureException(e1.getMessage()); } @@ -273,8 +300,19 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp try { repositoryExecutor.commitTransaction(); } catch (TransactionCommitFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentUploadFailureException(e.getMessage()); } + + StateService.getTransactionProcessorGuard().unlock(); + + StateService.getWatcherService().increaseFilesUploadCounter(); + + StateService.getWatcherService().increaseUploadedFilesSize(fileSize); + + telemetryService.setAverageUploadFileSizeQueue( + StateService.getWatcherService().getAverageFileSize()); } /** @@ -296,14 +334,26 @@ public byte[] downloadObject( String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); + TemporateContentUnitDto temporateContentUnit; + try { - if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, location)) { - return workspaceFacade.getObjectFile(workspaceUnitKey, location); - } - } catch (FileExistenceCheckFailureException | FileUnitRetrievalFailureException e) { + temporateContentUnit = + repositoryFacade.retrieveTemporateContentByLocationProviderAndSecret( + location, validationSecretsUnit); + } catch (TemporateContentRemovalFailureException e) { throw new ProcessorContentDownloadFailureException(e.getMessage()); } + if (Objects.nonNull(temporateContentUnit)) { + try { + if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, temporateContentUnit.getHash())) { + return workspaceFacade.getObjectFile(workspaceUnitKey, temporateContentUnit.getHash()); + } + } catch (FileExistenceCheckFailureException | FileUnitRetrievalFailureException e) { + throw new ProcessorContentDownloadFailureException(e.getMessage()); + } + } + RepositoryContentUnitDto repositoryContentLocationUnitDto; try { @@ -383,9 +433,13 @@ public void removeObject( throws ProcessorContentRemovalFailureException { logger.info(String.format("Removing content object of '%s' location", location)); + StateService.getTransactionProcessorGuard().lock(); + try { repositoryExecutor.beginTransaction(); } catch (TransactionInitializationFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e.getMessage()); } @@ -398,9 +452,13 @@ public void removeObject( try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } @@ -412,9 +470,13 @@ public void removeObject( try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } @@ -431,9 +493,13 @@ public void removeObject( try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } @@ -448,32 +514,66 @@ public void removeObject( try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } - } - try { - if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, location)) { - workspaceFacade.removeObjectFile(workspaceUnitKey, location); - } - } catch (FileExistenceCheckFailureException | FileRemovalFailureException e1) { + TemporateContentUnitDto temporateContentUnit; + try { - repositoryExecutor.rollbackTransaction(); - } catch (TransactionRollbackFailureException e2) { - throw new ProcessorContentRemovalFailureException(e2.getMessage()); + temporateContentUnit = + repositoryFacade.retrieveTemporateContentByLocationProviderAndSecret( + location, validationSecretsUnit); + } catch (TemporateContentRemovalFailureException e1) { + try { + repositoryExecutor.rollbackTransaction(); + } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e2.getMessage()); + } + + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } - throw new ProcessorContentRemovalFailureException(e1.getMessage()); + if (Objects.nonNull(temporateContentUnit)) { + try { + if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, temporateContentUnit.getHash())) { + workspaceFacade.removeObjectFile(workspaceUnitKey, temporateContentUnit.getHash()); + } + } catch (FileExistenceCheckFailureException | FileRemovalFailureException e1) { + try { + repositoryExecutor.rollbackTransaction(); + } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e2.getMessage()); + } + + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentRemovalFailureException(e1.getMessage()); + } + } } try { repositoryExecutor.commitTransaction(); } catch (TransactionCommitFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e.getMessage()); } + + StateService.getTransactionProcessorGuard().unlock(); } /** diff --git a/api-server/src/main/java/com/objectstorage/service/state/StateService.java b/api-server/src/main/java/com/objectstorage/service/state/StateService.java index b2317b1..40214b8 100644 --- a/api-server/src/main/java/com/objectstorage/service/state/StateService.java +++ b/api-server/src/main/java/com/objectstorage/service/state/StateService.java @@ -1,5 +1,6 @@ package com.objectstorage.service.state; +import com.objectstorage.service.state.watcher.WatcherService; import lombok.Getter; import lombok.Setter; @@ -27,4 +28,16 @@ public class StateService { */ @Getter private final static ReentrantLock backupProcessorGuard = new ReentrantLock(); + + /** + * Represents ObjectStorage transaction guard. + */ + @Getter + private final static ReentrantLock transactionProcessorGuard = new ReentrantLock(); + + /** + * Represents ObjectStorage watcher service instance. + */ + @Getter + private static final WatcherService watcherService = new WatcherService(); } diff --git a/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java b/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java new file mode 100644 index 0000000..bf3a21e --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java @@ -0,0 +1,47 @@ +package com.objectstorage.service.state.watcher; + +import org.apache.commons.io.FileUtils; + +/** + * Service used to track state metrics for the current session in the application. + */ +public class WatcherService { + /** + * Represents amount of files uploaded to ObjectStorage Temporate Storage in the current session. + */ + private Integer filesUploadCounter = 0; + + /** + * Increases amount of files uploaded to ObjectStorage Temporate Storage in the current session. + */ + public void increaseFilesUploadCounter() { + filesUploadCounter++; + } + + /** + * Represents global files size uploaded in the current session. + */ + private Integer uploadedFilesSize = 0; + + /** + * Increases global files size uploaded in the current session with the given value. + * + * @param value given value of uploaded file size. + */ + public void increaseUploadedFilesSize(Integer value) { + uploadedFilesSize += value; + } + + /** + * Calculates average file size in the current session. + * + * @return calculated average file size. + */ + public Double getAverageFileSize() { + if (filesUploadCounter > 0) { + return Double.valueOf(uploadedFilesSize) / Double.valueOf(filesUploadCounter) / 1024 / 1024; + } + + return (double) 0; + } +} diff --git a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java index 75b7be8..b8c7f50 100644 --- a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java +++ b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java @@ -35,6 +35,8 @@ public class TelemetryService { private final ConcurrentLinkedQueue averageUploadFileSizeQueue = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue currentBackupsAmountQueue = new ConcurrentLinkedQueue<>(); + private final static ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(0, Thread.ofVirtual().factory()); @@ -67,30 +69,22 @@ private void configure() { } }, 0, properties.getDiagnosticsScrapeDelay(), TimeUnit.MILLISECONDS); - System.out.println(configService.getConfig().getTemporateStorage().getFrequency()); - - telemetryBinding.getTemporateStorageFilesAmount().set(10); - } - - /** - * Increases current amount of files in ObjectStorage Temporate Storage. - */ - public void increaseTemporateStorageFilesAmount() { - if (configService.getConfig().getDiagnostics().getEnabled()) { - temporateStorageFilesAmountQueue.add( - () -> telemetryBinding.getTemporateStorageFilesAmount().set( - telemetryBinding.getTemporateStorageFilesAmount().get() + 1)); - } + scheduledExecutorService.scheduleWithFixedDelay(() -> { + if (!currentBackupsAmountQueue.isEmpty()) { + currentBackupsAmountQueue.poll().run(); + } + }, 0, properties.getDiagnosticsScrapeDelay(), TimeUnit.MILLISECONDS); } /** - * Decreases current amount of files in ObjectStorage Temporate Storage. + * Sets current amount of files in ObjectStorage Temporate Storage. + * + * @param value given value. */ - public void decreaseTemporateStorageFilesAmount() { + public void setTemporateStorageFilesAmount(Integer value) { if (configService.getConfig().getDiagnostics().getEnabled()) { temporateStorageFilesAmountQueue.add( - () -> telemetryBinding.getTemporateStorageFilesAmount().set( - telemetryBinding.getTemporateStorageFilesAmount().get() - 1)); + () -> telemetryBinding.getTemporateStorageFilesAmount().set(value)); } } @@ -105,17 +99,6 @@ public void increaseCurrentCloudServiceUploads() { } } - /** - * Decreases current cloud service uploads from ObjectStorage Temporate Storage. - */ - public void decreaseCurrentCloudServiceUploads() { - if (configService.getConfig().getDiagnostics().getEnabled()) { - currentCloudServiceUploadsQueue.add( - () -> telemetryBinding.getCurrentCloudServiceUploadsAmount().set( - telemetryBinding.getCurrentCloudServiceUploadsAmount().get() - 1)); - } - } - /** * Increases cloud service upload retries form ObjectStorage Temporate Storage. */ @@ -128,35 +111,25 @@ public void increaseCloudServiceUploadRetries() { } /** - * Decreases cloud service upload retries form ObjectStorage Temporate Storage. - */ - public void decreaseCloudServiceUploadRetries() { - if (configService.getConfig().getDiagnostics().getEnabled()) { - cloudServiceUploadRetriesQueue.add( - () -> telemetryBinding.getCloudServiceUploadRetries().set( - telemetryBinding.getCloudServiceUploadRetries().get() - 1)); - } - } - - /** - * Increases average upload file size to ObjectStorage Temporate Storage. + * Sets average upload file size to ObjectStorage Temporate Storage. + * + * @param value given value. */ - public void increaseAverageUploadFileSizeQueue() { + public void setAverageUploadFileSizeQueue(Double value) { if (configService.getConfig().getDiagnostics().getEnabled()) { averageUploadFileSizeQueue.add( - () -> telemetryBinding.getAverageUploadFileSize().set( - telemetryBinding.getAverageUploadFileSize().get() + 1)); + () -> telemetryBinding.getAverageUploadFileSize().set(value)); } } /** - * Decreases average upload file size to ObjectStorage Temporate Storage. + * Increases performed cloud service backup operations. */ - public void decreaseAverageUploadFileSizeQueue() { + public void increaseCurrentBackupsAmount() { if (configService.getConfig().getDiagnostics().getEnabled()) { - averageUploadFileSizeQueue.add( - () -> telemetryBinding.getAverageUploadFileSize().set( - telemetryBinding.getAverageUploadFileSize().get() - 1)); + currentBackupsAmountQueue.add( + () -> telemetryBinding.getCurrentBackupsAmount().set( + telemetryBinding.getCurrentBackupsAmount().get() + 1)); } } } diff --git a/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java b/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java index 79957ad..7b51ec1 100644 --- a/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java +++ b/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java @@ -1,5 +1,6 @@ package com.objectstorage.service.telemetry.binding; +import com.google.common.util.concurrent.AtomicDouble; import io.micrometer.core.instrument.Timer; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; @@ -9,6 +10,7 @@ import org.jetbrains.annotations.NotNull; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; /** * Service used to create custom telemetry bindings used to distribute application metrics. @@ -22,9 +24,11 @@ public class TelemetryBinding implements MeterBinder { private final AtomicInteger cloudServiceUploadRetries = new AtomicInteger(); - private final AtomicInteger configuredTemporateStorageAwaitTime = new AtomicInteger(); + private final AtomicLong configuredTemporateStorageAwaitTime = new AtomicLong(); - private final AtomicInteger averageUploadFileSize = new AtomicInteger(); + private final AtomicDouble averageUploadFileSize = new AtomicDouble(); + + private final AtomicInteger currentBackupsAmount = new AtomicInteger(); /** * @see MeterBinder @@ -36,19 +40,23 @@ public void bindTo(@NotNull MeterRegistry meterRegistry) { .register(meterRegistry); Gauge.builder("general.current_cloud_service_uploads_amount", currentCloudServiceUploadsAmount, AtomicInteger::get) - .description("Represents amount of uploads to different cloud services") + .description("Represents amount of uploads to different cloud services in the current session") .register(meterRegistry); Gauge.builder("general.cloud_service_upload_retries", cloudServiceUploadRetries, AtomicInteger::get) .description("Represents cloud service uploads retries from ObjectStorage Temporate Storage") .register(meterRegistry); - Gauge.builder("general.configured_temporate_storage_await_time", configuredTemporateStorageAwaitTime, AtomicInteger::get) + Gauge.builder("general.configured_temporate_storage_await_time", configuredTemporateStorageAwaitTime, AtomicLong::get) .description("Represents configured ObjectStorage Temporate Storage await time") .register(meterRegistry); - Gauge.builder("general.average_upload_file_size", averageUploadFileSize, AtomicInteger::get) + Gauge.builder("general.average_upload_file_size", averageUploadFileSize, AtomicDouble::get) .description("Represents average upload file size in ObjectStorage Temporate Storage") .register(meterRegistry); + + Gauge.builder("general.current_backups_amount", currentBackupsAmount, AtomicInteger::get) + .description("Represents amount of performed cloud service backup operation in the current session") + .register(meterRegistry); } } diff --git a/api-server/src/main/java/com/objectstorage/service/vendor/VendorFacade.java b/api-server/src/main/java/com/objectstorage/service/vendor/VendorFacade.java index 8751f64..930b273 100644 --- a/api-server/src/main/java/com/objectstorage/service/vendor/VendorFacade.java +++ b/api-server/src/main/java/com/objectstorage/service/vendor/VendorFacade.java @@ -319,7 +319,8 @@ public List listAllObjectsFromBucket( bucketName, credentialsFieldExternal.getRegion()) .stream() - .map(ContentRetrievalProviderUnit::of) + .map(element -> ContentRetrievalProviderUnit.of( + element.getLocation(), element.getCreatedAt())) .toList(); } case GCS -> { @@ -333,7 +334,8 @@ public List listAllObjectsFromBucket( yield gcsVendorService.listObjectsFromGCSBucket(credentials, bucketName) .stream() - .map(ContentRetrievalProviderUnit::of) + .map(element -> ContentRetrievalProviderUnit.of( + element.getLocation(), element.getCreatedAt())) .toList(); } }; diff --git a/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java b/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java index b03d84a..2a5acea 100644 --- a/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java +++ b/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java @@ -8,6 +8,7 @@ import com.google.cloud.resourcemanager.ResourceManagerOptions; import com.google.cloud.resourcemanager.Project; import com.google.cloud.storage.*; +import com.objectstorage.dto.VendorObjectListingDto; import com.objectstorage.exception.GCPCredentialsInitializationFailureException; import com.objectstorage.exception.GCSBucketObjectUploadFailureException; import jakarta.enterprise.context.ApplicationScoped; @@ -173,7 +174,7 @@ public byte[] retrieveObjectFromGCSBucket( * @param bucketName given name of the GCS bucket. * @return listed objects. */ - public List listObjectsFromGCSBucket( + public List listObjectsFromGCSBucket( Credentials credentials, String bucketName) { Storage storage = StorageOptions.newBuilder() @@ -184,7 +185,9 @@ public List listObjectsFromGCSBucket( Page blobs = storage.list(bucketName); return StreamSupport.stream(blobs.iterateAll().spliterator(), false) - .map(element -> element.getBlobId().getName()) + .map(element -> VendorObjectListingDto.of( + element.getBlobId().getName(), + element.getUpdateTimeOffsetDateTime().toEpochSecond())) .toList(); } diff --git a/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java b/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java index c916537..d8ea071 100644 --- a/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java +++ b/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java @@ -12,6 +12,7 @@ import com.amazonaws.services.securitytoken.model.GetCallerIdentityRequest; import com.amazonaws.waiters.WaiterParameters; import com.objectstorage.dto.AWSSecretsDto; +import com.objectstorage.dto.VendorObjectListingDto; import com.objectstorage.exception.S3BucketObjectRetrievalFailureException; import com.objectstorage.exception.VendorOperationFailureException; import jakarta.enterprise.context.ApplicationScoped; @@ -172,6 +173,11 @@ public void uploadObjectToS3Bucket( put("objectstorage", "true"); } }); + try { + metadata.setContentLength(inputStream.available()); + } catch (IOException e) { + throw new VendorOperationFailureException(e.getMessage()); + } PutObjectRequest request = new PutObjectRequest(bucketName, fileName, inputStream, metadata); @@ -266,7 +272,7 @@ public byte[] retrieveObjectFromS3Bucket( * @return listed objects. * @throws VendorOperationFailureException if vendor operation fails. */ - public List listObjectsFromS3Bucket( + public List listObjectsFromS3Bucket( AWSCredentialsProvider awsCredentialsProvider, String bucketName, String region) throws VendorOperationFailureException { @@ -280,7 +286,8 @@ public List listObjectsFromS3Bucket( return simpleStorage .listObjects(bucketName) .getObjectSummaries() - .stream().map(S3ObjectSummary::getKey) + .stream().map(element -> VendorObjectListingDto.of( + element.getKey(), element.getLastModified().getTime())) .toList(); } catch (Exception e) { throw new VendorOperationFailureException(e.getMessage()); diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java b/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java index a9da9a8..b59b925 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java @@ -1,8 +1,10 @@ package com.objectstorage.service.workspace; +import com.objectstorage.dto.FolderContentUnitDto; import com.objectstorage.entity.common.PropertiesEntity; import com.objectstorage.exception.*; import com.objectstorage.exception.FileNotFoundException; +import com.objectstorage.service.workspace.common.WorkspaceConfigurationHelper; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import jakarta.xml.bind.DatatypeConverter; @@ -19,8 +21,8 @@ import java.util.List; import java.util.Objects; import java.util.stream.Stream; +import java.util.zip.Deflater; import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; import lombok.SneakyThrows; @@ -92,7 +94,7 @@ public void createContentDirectory(String workspaceUnitDirectory, String type) t * @param key given workspace unit directory. * @return result of the check. */ - public Boolean isUnitDirectoryExist(String key) { + private Boolean isUnitDirectoryExist(String key) { return Files.exists(Paths.get(properties.getWorkspaceDirectory(), key)); } @@ -103,7 +105,7 @@ public Boolean isUnitDirectoryExist(String key) { * @param type given content directory type. * @return result of the check. */ - public Boolean isContentDirectoryExist(String workspaceUnitDirectory, String type) { + private Boolean isContentDirectoryExist(String workspaceUnitDirectory, String type) { return Files.exists(Paths.get(workspaceUnitDirectory, type)); } @@ -316,16 +318,56 @@ public byte[] compressFile(InputStream inputStream) throws return result.toByteArray(); } + /** + * Compresses given folder entities of the given type. This will act as a non-compressed folder, which has + * previously compressed with zip files. + * + * @param folderContentUnits given folder input entities. + * @param type given folder type. + * @return compressed folder input. + * @throws InputCompressionFailureException if input compression fails. + */ + public byte[] compressFolder(List folderContentUnits, String type) throws + InputCompressionFailureException { + ByteArrayOutputStream result = new ByteArrayOutputStream(); + + try (ZipOutputStream writer = new ZipOutputStream(result)) { + writer.setMethod(ZipOutputStream.DEFLATED); + writer.setLevel(Deflater.NO_COMPRESSION); + + writer.putNextEntry(new ZipEntry(WorkspaceConfigurationHelper.getZipFolderDefinition(type))); + + for (FolderContentUnitDto folderContentUnit : folderContentUnits) { + writer.putNextEntry(new ZipEntry( + Path.of( + properties.getWorkspaceContentBackupDirectory(), + WorkspaceConfigurationHelper.getZipFile( + folderContentUnit.getLocation())).toString())); + + writer.write(folderContentUnit.getContent()); + } + + writer.flush(); + + writer.finish(); + + } catch (IOException e) { + throw new InputCompressionFailureException(e.getMessage()); + } + + return result.toByteArray(); + } + /** * Adds new file of the given type to the workspace with the given workspace unit key as the compressed input stream. * * @param workspaceUnitKey given user workspace unit key. * @param type given content type. * @param name given content name. - * @param inputStream given input. + * @param content given content. * @throws FileCreationFailureException if file creation operation failed. */ - public void addContentFile(String workspaceUnitKey, String type, String name, InputStream inputStream) + public void addContentFile(String workspaceUnitKey, String type, String name, byte[] content) throws FileCreationFailureException { if (!isUnitDirectoryExist(workspaceUnitKey)) { try { @@ -355,14 +397,6 @@ public void addContentFile(String workspaceUnitKey, String type, String name, In throw new FileCreationFailureException(); } - byte[] content; - - try { - content = compressFile(inputStream); - } catch (InputCompressionFailureException e) { - throw new FileCreationFailureException(e.getMessage()); - } - try { createFile(workspaceUnitDirectory, type, name, content); } catch (FileWriteFailureException e) { @@ -396,6 +430,39 @@ public Boolean isContentFilePresent(String workspaceUnitKey, String type, String return false; } + /** + * Retrieves content units in the workspace with the given workspace unit key and of the given type. + * + * @param workspaceUnitKey given user workspace unit key. + * @param type given file type. + * @return retrieves additional content units. + * @throws FileUnitsRetrievalFailureException if content units retrieval failed. + */ + public List getContentUnits(String workspaceUnitKey, String type) throws + FileUnitsRetrievalFailureException { + List result = new ArrayList<>(); + + if (isUnitDirectoryExist(workspaceUnitKey)) { + String workspaceUnitDirectory; + + try { + workspaceUnitDirectory = getUnitDirectory(workspaceUnitKey); + } catch (WorkspaceUnitDirectoryNotFoundException e) { + throw new FileUnitsRetrievalFailureException(e.getMessage()); + } + + if (isContentDirectoryExist(workspaceUnitDirectory, type)) { + try { + result = getFilesLocations(workspaceUnitDirectory, type); + } catch (FilesLocationsRetrievalFailureException e) { + throw new FileUnitsRetrievalFailureException(e.getMessage()); + } + } + } + + return result; + } + /** * Retrieves file from the workspace with the given workspace unit key as compressed byte array. * diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java b/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java new file mode 100644 index 0000000..f95e0de --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java @@ -0,0 +1,27 @@ +package com.objectstorage.service.workspace.common; + +/** + * Contains helpful tools used for workspace configuration. + */ +public class WorkspaceConfigurationHelper { + + /** + * Creates folder definition with the help of the given folder name for ZIP achieve. + * + * @param name given folder name value. + * @return wrapped token. + */ + public static String getZipFolderDefinition(String name) { + return String.format("%s/", name); + } + + /** + * Creates zip file definition with the help of the given file name. + * + * @param name given zip file name. + * @return wrapped zip file. + */ + public static String getZipFile(String name) { + return String.format("%s.zip", name); + } +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java index 35e010a..6d90ec5 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java @@ -1,7 +1,9 @@ package com.objectstorage.service.workspace.facade; +import com.objectstorage.dto.FolderContentUnitDto; import com.objectstorage.entity.common.PropertiesEntity; import com.objectstorage.exception.*; +import com.objectstorage.model.ContentRetrievalBackupUnit; import com.objectstorage.model.ValidationSecretsApplication; import com.objectstorage.service.config.ConfigService; import com.objectstorage.service.workspace.WorkspaceService; @@ -10,6 +12,7 @@ import java.io.*; import java.time.Instant; +import java.util.List; import java.util.stream.Collectors; /** @@ -55,7 +58,12 @@ public String createWorkspaceUnitKey(ValidationSecretsApplication validationSecr * @return created file unit key. */ public String createFileUnitKey(String name) { - return workspaceService.createUnitKey(name, Instant.now().toString()); + Instant timestamp = Instant.now(); + + String fileUnit = + workspaceService.createUnitKey(name, Instant.now().toString()); + + return String.format("%s-%s-%d", name, fileUnit, timestamp.toEpochMilli()); } /** @@ -68,20 +76,45 @@ public String createFileUnitKey(String name) { */ public void addObjectFile(String workspaceUnitKey, String name, InputStream inputStream) throws FileCreationFailureException { - workspaceService.addContentFile(workspaceUnitKey, properties.getWorkspaceContentObjectDirectory(), name, inputStream); + byte[] content; + + try { + content = workspaceService.compressFile(inputStream); + } catch (InputCompressionFailureException e) { + throw new FileCreationFailureException(e.getMessage()); + } + + workspaceService.addContentFile( + workspaceUnitKey, + properties.getWorkspaceContentObjectDirectory(), + name, + content); } /** * Adds new backup file to the workspace with the given workspace unit key as the compressed input stream. * * @param workspaceUnitKey given user workspace unit key. - * @param name given content name. - * @param inputStream given input. + * @param name given file name. + * @param folderContentUnits given folder content units. * @throws FileCreationFailureException if file creation operation failed. */ - public void addBackupFile(String workspaceUnitKey, String name, InputStream inputStream) + public void addBackupFile(String workspaceUnitKey, String name, List folderContentUnits) throws FileCreationFailureException { - workspaceService.addContentFile(workspaceUnitKey, properties.getWorkspaceContentBackupDirectory(), name, inputStream); + byte[] content; + + try { + content = + workspaceService.compressFolder(folderContentUnits, properties.getWorkspaceContentBackupDirectory()); + } catch (InputCompressionFailureException e) { + throw new FileCreationFailureException(e.getMessage()); + } + + workspaceService.addContentFile( + workspaceUnitKey, + properties.getWorkspaceContentBackupDirectory(), + name, + content); Integer amount; @@ -130,6 +163,21 @@ public Boolean isBackupFilePresent(String workspaceUnitKey, String name) throws workspaceUnitKey, properties.getWorkspaceContentBackupDirectory(), name); } + /** + * Retrieves backup units from the workspace with the given workspace unit key. + * + * @param workspaceUnitKey given user workspace unit key. + * @return retrieved backup units. + * @throws FileUnitsRetrievalFailureException if file units retrieval fails. + */ + public List getBackupUnits(String workspaceUnitKey) throws FileUnitsRetrievalFailureException { + return workspaceService + .getContentUnits(workspaceUnitKey, properties.getWorkspaceContentBackupDirectory()) + .stream() + .map(ContentRetrievalBackupUnit::of) + .toList(); + } + /** * Retrieves object file with the given name and of the given type from the workspace with the given workspace * unit key as compressed byte array. @@ -166,16 +214,6 @@ public void removeObjectFile(String workspaceUnitKey, String name) throws FileRe workspaceService.removeContentFile(workspaceUnitKey, properties.getWorkspaceContentObjectDirectory(), name); } - /** - * Removes backup file with the given name from the workspace with the help of the given workspace unit key. - * - * @param workspaceUnitKey given user workspace unit key. - * @throws FileRemovalFailureException if file removal operation failed. - */ - public void removeBackupFile(String workspaceUnitKey, String name) throws FileRemovalFailureException { - workspaceService.removeContentFile(workspaceUnitKey, properties.getWorkspaceContentBackupDirectory(), name); - } - /** * Removes all the files from the workspace with the help of the given workspace unit key. * diff --git a/api-server/src/main/openapi/openapi.yml b/api-server/src/main/openapi/openapi.yml index 136b787..f6092e3 100644 --- a/api-server/src/main/openapi/openapi.yml +++ b/api-server/src/main/openapi/openapi.yml @@ -264,6 +264,7 @@ components: required: - pending - uploaded + - backups properties: pending: type: array @@ -273,7 +274,21 @@ components: type: array items: $ref: "#/components/schemas/ContentRetrievalProviderUnit" + backups: + type: array + items: + $ref: "#/components/schemas/ContentRetrievalBackupUnit" ContentRetrievalProviderUnit: + required: + - location + - created_at + properties: + location: + type: string + created_at: + type: integer + format: int64 + ContentRetrievalBackupUnit: required: - location properties: diff --git a/api-server/src/main/resources/application.properties b/api-server/src/main/resources/application.properties index f7dd05d..d9213f4 100644 --- a/api-server/src/main/resources/application.properties +++ b/api-server/src/main/resources/application.properties @@ -74,6 +74,9 @@ workspace.content.object.directory=object # Describes location of backup content directory. workspace.content.backup.directory=backup +# Describes location of backup content unit. +workspace.content.backup.unit=backup + # Describes name of the file used for compression operation. workspace.compression.file.name=objectstorage-file diff --git a/config/grafana/dashboards/diagnostics.tmpl b/config/grafana/dashboards/diagnostics.tmpl index 8d6ee04..a9e7da5 100644 --- a/config/grafana/dashboards/diagnostics.tmpl +++ b/config/grafana/dashboards/diagnostics.tmpl @@ -20,7 +20,6 @@ "fiscalYearStartMonth": 0, "gnetId": 179, "graphTooltip": 1, - "id": 1, "links": [], "panels": [ { @@ -31,120 +30,6 @@ "x": 0, "y": 0 }, - "id": 30, - "panels": [], - "title": "ObjectStorage Cluster", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P21B111CBFE6E8FCA" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 38, - "interval": "500ms", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P21B111CBFE6E8FCA" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "general_cluster_download_amount", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "ObjectStorage Cluster content download ongoing requests", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, "id": 29, "panels": [], "title": "ObjectStorage API Server", @@ -214,7 +99,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 1 }, "id": 36, "interval": "500ms", @@ -238,7 +123,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "general_raw_content_upload_amount", + "expr": "general_temporate_storage_files_amount", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -248,7 +133,7 @@ "useBackend": false } ], - "title": "ObjectStorage API Server raw content upload ongoing requests", + "title": "ObjectStorage API Server Temporate Storage current files amount", "type": "timeseries" }, { @@ -315,7 +200,7 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 1 }, "id": 37, "interval": "500ms", @@ -339,7 +224,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "general_additional_content_upload_amount", + "expr": "general_current_cloud_service_uploads_amount", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -349,7 +234,7 @@ "useBackend": false } ], - "title": "ObjectStorage API Server additional content upload ongoing requests", + "title": "ObjectStorage API Server current cloud service uploads amount", "type": "timeseries" }, { @@ -416,7 +301,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 9 }, "id": 33, "interval": "0.1", @@ -440,7 +325,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "general_api_server_health_check_amount", + "expr": "general_cloud_service_upload_retries", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -450,7 +335,7 @@ "useBackend": false } ], - "title": "ObjectStorage API Server healthcheck ongoing requests", + "title": "ObjectStorage API Server cloud service upload retries", "type": "timeseries" }, { @@ -509,7 +394,8 @@ "value": 80 } ] - } + }, + "unit": "decmbytes" }, "overrides": [] }, @@ -517,7 +403,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 9 }, "id": 34, "interval": "0.1", @@ -541,7 +427,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "general_cluster_health_check_amount", + "expr": "general_average_upload_file_size", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -551,7 +437,7 @@ "useBackend": false } ], - "title": "ObjectStorage Cluster healthcheck requests", + "title": "ObjectStorage API Server average upload file size", "type": "timeseries" }, { @@ -580,9 +466,9 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "stepBefore", + "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 6, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -597,7 +483,6 @@ } }, "decimals": 0, - "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", @@ -611,7 +496,8 @@ "value": 80 } ] - } + }, + "unit": "decmbytes" }, "overrides": [] }, @@ -619,9 +505,9 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 17 }, - "id": 31, + "id": 38, "interval": "0.1", "options": { "legend": { @@ -643,7 +529,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "general_serving_cluster_amount", + "expr": "general_current_backups_amount", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -653,7 +539,7 @@ "useBackend": false } ], - "title": "Serving ObjectStorage Cluster allocations", + "title": "ObjectStorage API Server current backups amount", "type": "timeseries" }, { @@ -663,42 +549,8 @@ }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepBefore", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, "decimals": 0, + "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", @@ -706,13 +558,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [] }, @@ -720,22 +569,27 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 17 }, "id": 32, "interval": "0.1", "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -744,7 +598,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "general_suspended_cluster_amount", + "expr": "general_configured_temporate_storage_await_time", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -754,8 +608,8 @@ "useBackend": false } ], - "title": "Suspended ObjectStorage Cluster allocations", - "type": "timeseries" + "title": "ObjectStorage API Server Temporate Storage configured await time(milliseconds)", + "type": "stat" }, { "datasource": { @@ -787,7 +641,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 25 }, "id": 35, "options": { @@ -835,7 +689,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 33 }, "id": 28, "panels": [], @@ -884,7 +738,7 @@ "h": 7, "w": 3, "x": 0, - "y": 43 + "y": 34 }, "id": 15, "maxDataPoints": 100, @@ -969,7 +823,7 @@ "h": 6, "w": 6, "x": 3, - "y": 43 + "y": 34 }, "id": 6, "maxDataPoints": 100, @@ -1055,7 +909,7 @@ "h": 6, "w": 6, "x": 9, - "y": 43 + "y": 34 }, "id": 4, "maxDataPoints": 100, @@ -1093,7 +947,7 @@ "type": "gauge" } ], - "refresh": "5s", + "refresh": "", "schemaVersion": 39, "tags": [ "docker", @@ -1269,8 +1123,8 @@ ] }, "time": { - "from": "now-5m", - "to": "now" + "from": "2024-11-29T14:23:10.911Z", + "to": "2024-11-29T14:25:03.115Z" }, "timepicker": { "refresh_intervals": [ diff --git a/samples/config/api-server/api-server.yaml b/samples/config/api-server/api-server.yaml index f1f2c63..4022b0a 100644 --- a/samples/config/api-server/api-server.yaml +++ b/samples/config/api-server/api-server.yaml @@ -17,9 +17,6 @@ connection: # Represents section used for ObjectStorage API Server temporate storage configuration. Same compression will be # used to upload files to the configured cloud providers. temporate-storage: - # Represents format used for content to be saved. - format: "zip" - # Represents frequency of scheduled operations processing. frequency: "*/5 * * * * ?" @@ -37,7 +34,7 @@ backup: # Represents section used for ObjectStorage API Server diagnostics configuration. diagnostics: # Enables diagnostics functionality. - enabled: false + enabled: true # Represents section used for ObjectStorage diagnostics metrics configuration. metrics: