From ce17d9762690554cfca613f287df15108d825351 Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi Date: Thu, 28 Nov 2024 09:20:45 +0100 Subject: [PATCH 1/9] fix: fixed bugs --- ...ValidationSecretsApplicationConverter.java | 32 +++++++++++++++++++ .../dto/ContentCompoundUnitDto.java | 29 +++++++++++++++++ .../dto/EarliestTemporateContentDto.java | 6 ++-- .../dto/RepositoryTemporateUnitDto.java | 23 ------------- .../repository/ContentRepository.java | 14 +++----- .../repository/facade/RepositoryFacade.java | 32 ++++++++++++++----- .../integration/backup/BackupService.java | 6 ++++ .../TemporateStorageService.java | 19 +++++++---- 8 files changed, 111 insertions(+), 50 deletions(-) create mode 100644 api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitToValidationSecretsApplicationConverter.java create mode 100644 api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java delete mode 100644 api-server/src/main/java/com/objectstorage/dto/RepositoryTemporateUnitDto.java diff --git a/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitToValidationSecretsApplicationConverter.java b/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitToValidationSecretsApplicationConverter.java new file mode 100644 index 0000000..84ec9c2 --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitToValidationSecretsApplicationConverter.java @@ -0,0 +1,32 @@ +package com.objectstorage.converter; + +import com.objectstorage.dto.ContentCompoundUnitDto; +import com.objectstorage.model.ValidationSecretsApplication; +import com.objectstorage.model.ValidationSecretsUnit; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents content compounds units to validation secrets application converter; + */ +public class ContentCompoundUnitToValidationSecretsApplicationConverter { + + /** + * Converts given content compound units to validation secrets application. + * + * @param contentCompoundUnits given content compounds units. + * @return converted validation secrets application. + */ + public static ValidationSecretsApplication convert(List contentCompoundUnits) { + List validationSecretsUnits = new ArrayList<>(); + + contentCompoundUnits.forEach( + element -> validationSecretsUnits.add( + ValidationSecretsUnit.of( + element.getProvider(), + element.getCredentialsFieldsFull()))); + + return ValidationSecretsApplication.of(validationSecretsUnits); + } +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java new file mode 100644 index 0000000..6a944ac --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java @@ -0,0 +1,29 @@ +package com.objectstorage.dto; + +import com.objectstorage.model.CredentialsFieldsFull; +import com.objectstorage.model.Provider; +import com.objectstorage.model.ValidationSecretsUnit; +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * Represents content compound unit dto. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class ContentCompoundUnitDto { + /** + * Represents root location for internal file system. + */ + private RepositoryContentUnitDto repositoryContentUnitDto; + + /** + * Represents provider. + */ + private Provider provider; + + /** + * Represents full credentials fields. + */ + private CredentialsFieldsFull credentialsFieldsFull; +} diff --git a/api-server/src/main/java/com/objectstorage/dto/EarliestTemporateContentDto.java b/api-server/src/main/java/com/objectstorage/dto/EarliestTemporateContentDto.java index 86ef950..a58e9cb 100644 --- a/api-server/src/main/java/com/objectstorage/dto/EarliestTemporateContentDto.java +++ b/api-server/src/main/java/com/objectstorage/dto/EarliestTemporateContentDto.java @@ -6,6 +6,8 @@ import lombok.AllArgsConstructor; import lombok.Getter; +import java.util.List; + /** * Represents dto used to represent the earliest temporate content. */ @@ -13,9 +15,9 @@ @AllArgsConstructor(staticName = "of") public class EarliestTemporateContentDto { /** - * Represents target provider. + * Represents content compound units. */ - private ValidationSecretsApplication validationSecretsApplication; + private List contentCompoundUnits; /** * Represents file location. diff --git a/api-server/src/main/java/com/objectstorage/dto/RepositoryTemporateUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/RepositoryTemporateUnitDto.java deleted file mode 100644 index c000c56..0000000 --- a/api-server/src/main/java/com/objectstorage/dto/RepositoryTemporateUnitDto.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.objectstorage.dto; - -import com.objectstorage.model.CredentialsFieldsFull; -import com.objectstorage.model.Provider; -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * Represents repository temporate unit. - */ -@Getter -@AllArgsConstructor(staticName = "of") -public class RepositoryTemporateUnitDto { - /** - * Represents file location. - */ - private String location; - - /** - * Represents selected service provider. - */ - private Provider provider; -} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java b/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java index 6fc91d7..8d4fda1 100644 --- a/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java +++ b/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java @@ -60,7 +60,7 @@ public void insert(Integer provider, Integer secret, String root) throws Reposit * @return retrieved content entities. * @throws RepositoryOperationFailureException if repository operation fails. */ - public List findByProviderAndSecret(Integer provider, Integer secret) throws + public ContentEntity findByProviderAndSecret(Integer provider, Integer secret) throws RepositoryOperationFailureException { ResultSet resultSet; @@ -77,18 +77,12 @@ public List findByProviderAndSecret(Integer provider, Integer sec throw new RepositoryOperationFailureException(e.getMessage()); } - List result = new ArrayList<>(); - Integer id; String root; try { - while (resultSet.next()) { - id = resultSet.getInt("id"); - root = resultSet.getString("root"); - - result.add(ContentEntity.of(id, provider, secret, root)); - } + id = resultSet.getInt("id"); + root = resultSet.getString("root"); } catch (SQLException e1) { try { resultSet.close(); @@ -105,7 +99,7 @@ public List findByProviderAndSecret(Integer provider, Integer sec throw new RepositoryOperationFailureException(e.getMessage()); } - return result; + return ContentEntity.of(id, provider, secret, root); } /** diff --git a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java index e110177..78e85be 100644 --- a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java +++ b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java @@ -1,7 +1,9 @@ package com.objectstorage.repository.facade; +import com.objectstorage.dto.ContentCompoundUnitDto; import com.objectstorage.dto.RepositoryContentUnitDto; import com.objectstorage.dto.EarliestTemporateContentDto; +import com.objectstorage.entity.repository.ContentEntity; import com.objectstorage.entity.repository.ProviderEntity; import com.objectstorage.entity.repository.SecretEntity; import com.objectstorage.entity.repository.TemporateEntity; @@ -124,7 +126,7 @@ public EarliestTemporateContentDto retrieveEarliestTemporateContent() throws Tem throw new TemporateContentRetrievalFailureException(e.getMessage()); } - List validationSecretsUnits = new ArrayList<>(); + List contentCompoundUnits = new ArrayList<>(); for (TemporateEntity temporate : temporateEntities) { ProviderEntity rawProvider; @@ -152,11 +154,24 @@ public EarliestTemporateContentDto retrieveEarliestTemporateContent() throws Tem secret.getSession(), secret.getCredentials()); - validationSecretsUnits.add(ValidationSecretsUnit.of(provider, secrets)); + ContentEntity contentEntity; + + try { + contentEntity = contentRepository.findByProviderAndSecret(rawProvider.getId(), secret.getId()); + } catch (RepositoryOperationFailureException e) { + throw new TemporateContentRetrievalFailureException(e.getMessage()); + } + + contentCompoundUnits.add( + ContentCompoundUnitDto.of( + RepositoryContentUnitDto.of( + contentEntity.getRoot()), + provider, + secrets)); } return EarliestTemporateContentDto.of( - ValidationSecretsApplication.of(validationSecretsUnits), + contentCompoundUnits, temporateEntity.getLocation(), temporateEntity.getHash(), temporateEntity.getCreatedAt()); @@ -203,15 +218,16 @@ public RepositoryContentUnitDto retrieveContentApplication(ValidationSecretsUnit throw new ContentApplicationRetrievalFailureException(e.getMessage()); } + ContentEntity contentEntity; + try { - return contentRepository - .findByProviderAndSecret(provider.getId(), secret.getId()) - .stream() - .map(element -> RepositoryContentUnitDto.of(element.getRoot())) - .toList().getFirst(); + contentEntity = contentRepository + .findByProviderAndSecret(provider.getId(), secret.getId()); } catch (RepositoryOperationFailureException e) { throw new ContentApplicationRetrievalFailureException(e.getMessage()); } + + return RepositoryContentUnitDto.of(contentEntity.getRoot()); } /** diff --git a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java index 7835a05..cc207d7 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java @@ -4,6 +4,7 @@ import com.objectstorage.exception.BackupPeriodRetrievalFailureException; import com.objectstorage.exception.CronExpressionException; import com.objectstorage.service.state.StateService; +import com.objectstorage.service.vendor.VendorFacade; import com.objectstorage.service.workspace.facade.WorkspaceFacade; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; @@ -27,6 +28,9 @@ public class BackupService { @Inject WorkspaceFacade workspaceFacade; + @Inject + VendorFacade vendorFacade; + private final ScheduledExecutorService scheduledOperationExecutorService = Executors.newSingleThreadScheduledExecutor(); @@ -49,7 +53,9 @@ public void process() throws BackupPeriodRetrievalFailureException { scheduledOperationExecutorService.scheduleWithFixedDelay(() -> { StateService.getBackupProcessorGuard().lock(); +// vendorFacade.listAllObjectsFromBucket().get(0). +// workspaceFacade.add StateService.getBackupProcessorGuard().unlock(); }, 0, period, TimeUnit.MILLISECONDS); diff --git a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java index bd63588..8a7dcd1 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java @@ -1,8 +1,11 @@ package com.objectstorage.service.integration.temporatestorage; +import com.objectstorage.converter.ContentCompoundUnitToValidationSecretsApplicationConverter; import com.objectstorage.converter.CronExpressionConverter; +import com.objectstorage.dto.ContentCompoundUnitDto; import com.objectstorage.dto.EarliestTemporateContentDto; import com.objectstorage.exception.*; +import com.objectstorage.model.ValidationSecretsApplication; import com.objectstorage.model.ValidationSecretsUnit; import com.objectstorage.repository.executor.RepositoryExecutor; import com.objectstorage.repository.facade.RepositoryFacade; @@ -122,8 +125,11 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { throw new RuntimeException(e1); } - String workspaceUnitKey = - workspaceFacade.createWorkspaceUnitKey(temporateContentDto.getValidationSecretsApplication()); + ValidationSecretsApplication validationSecretsApplication = + ContentCompoundUnitToValidationSecretsApplicationConverter.convert( + temporateContentDto.getContentCompoundUnits()); + + String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); byte[] content; @@ -145,13 +151,12 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { throw new RuntimeException(e1); } - for (ValidationSecretsUnit validationSecretsUnit : temporateContentDto.getValidationSecretsApplication() - .getSecrets()) { + for (ContentCompoundUnitDto contentCompoundUnit : temporateContentDto.getContentCompoundUnits()) { try { vendorFacade.uploadObjectToBucket( - validationSecretsUnit.getProvider(), - validationSecretsUnit.getCredentials().getExternal(), - "", + contentCompoundUnit.getProvider(), + contentCompoundUnit.getCredentialsFieldsFull().getExternal(), + contentCompoundUnit.getRepositoryContentUnitDto().getRoot(), temporateContentDto.getLocation(), new ByteArrayInputStream(content)); } catch ( From 7be609a7979871c147188b546f66acbe9f1c5f6e Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi Date: Thu, 28 Nov 2024 12:14:06 +0100 Subject: [PATCH 2/9] fix: fixed bugs --- .../dto/TemporateContentUnitDto.java | 36 +++++++++++ .../repository/TemporateRepository.java | 58 +++++++++++++++++ .../repository/facade/RepositoryFacade.java | 64 ++++++++++++++++++- .../integration/backup/BackupService.java | 4 ++ .../TemporateStorageService.java | 4 +- .../service/processor/ProcessorService.java | 51 +++++++++++---- .../service/telemetry/TelemetryService.java | 2 - .../service/vendor/s3/S3VendorService.java | 5 ++ api-server/src/main/openapi/openapi.yml | 4 ++ 9 files changed, 211 insertions(+), 17 deletions(-) create mode 100644 api-server/src/main/java/com/objectstorage/dto/TemporateContentUnitDto.java diff --git a/api-server/src/main/java/com/objectstorage/dto/TemporateContentUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/TemporateContentUnitDto.java new file mode 100644 index 0000000..720053e --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/TemporateContentUnitDto.java @@ -0,0 +1,36 @@ +package com.objectstorage.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * Represents repository content unit. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class TemporateContentUnitDto { + /** + * Represents provider. + */ + private Integer provider; + + /** + * Represents se. + */ + private Integer secret; + + /** + * Represents file location. + */ + private String location; + + /** + * Represents file hash. + */ + private String hash; + + /** + * Represents created at timestamp. + */ + private Long createdAt; +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/repository/TemporateRepository.java b/api-server/src/main/java/com/objectstorage/repository/TemporateRepository.java index 2030840..5069417 100644 --- a/api-server/src/main/java/com/objectstorage/repository/TemporateRepository.java +++ b/api-server/src/main/java/com/objectstorage/repository/TemporateRepository.java @@ -156,6 +156,7 @@ public TemporateEntity findEarliest() throws RepositoryOperationFailureException /** * Retrieves all the persisted temporate entities with the given hash. * + * @param hash given hash. * @return retrieved temporate entities. * @throws RepositoryOperationFailureException if repository operation fails. */ @@ -212,9 +213,66 @@ public List findByHash(String hash) throws return result; } + /** + * Retrieves all the persisted temporate entities with the given location, provider and secret. + * + * @param location given location. + * @param provider given provider. + * @param secret given secret. + * @return retrieved temporate entity. + * @throws RepositoryOperationFailureException if repository operation fails. + */ + public TemporateEntity findEarliestByLocationProviderAndSecret( + String location, Integer provider, Integer secret) throws RepositoryOperationFailureException { + ResultSet resultSet; + + try { + resultSet = + repositoryExecutor.performQueryWithResult( + String.format( + "SELECT t.id, t.hash, t.created_at FROM %s as t WHERE t.location = '%s' AND t.provider = %d AND t.secret = %d ORDER BY created_at DESC LIMIT 1", + properties.getDatabaseTemporateTableName(), + location, + provider, + secret)); + + } catch (QueryExecutionFailureException | QueryEmptyResultException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + Integer id; + String hash; + Long createdAt; + + try { + id = resultSet.getInt("id"); + hash = resultSet.getString("hash"); + createdAt = resultSet.getLong("created_at"); + + } catch (SQLException e1) { + try { + resultSet.close(); + } catch (SQLException e2) { + throw new RepositoryOperationFailureException(e2.getMessage()); + } + + throw new RepositoryOperationFailureException(e1.getMessage()); + } + + try { + resultSet.close(); + } catch (SQLException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + return TemporateEntity.of(id, provider, secret, location, hash, createdAt); + } + /** * Retrieves all the persisted temporate entities with the given provider and secret. * + * @param provider given provider. + * @param secret given secret. * @return retrieved temporate entities. * @throws RepositoryOperationFailureException if repository operation fails. */ diff --git a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java index 78e85be..771fa09 100644 --- a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java +++ b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java @@ -3,6 +3,7 @@ import com.objectstorage.dto.ContentCompoundUnitDto; import com.objectstorage.dto.RepositoryContentUnitDto; import com.objectstorage.dto.EarliestTemporateContentDto; +import com.objectstorage.dto.TemporateContentUnitDto; import com.objectstorage.entity.repository.ContentEntity; import com.objectstorage.entity.repository.ProviderEntity; import com.objectstorage.entity.repository.SecretEntity; @@ -82,7 +83,8 @@ public List retrieveFilteredTemporateContent( } return temporateContent.stream().map( - element -> ContentRetrievalProviderUnit.of(element.getLocation())).toList(); + element -> ContentRetrievalProviderUnit.of( + element.getLocation(), element.getCreatedAt())).toList(); } /** @@ -143,7 +145,7 @@ public EarliestTemporateContentDto retrieveEarliestTemporateContent() throws Tem SecretEntity secret; try { - secret = secretRepository.findById(temporate.getId()); + secret = secretRepository.findById(temporate.getSecret()); } catch (RepositoryOperationFailureException e) { throw new TemporateContentRetrievalFailureException(e.getMessage()); } @@ -177,6 +179,64 @@ public EarliestTemporateContentDto retrieveEarliestTemporateContent() throws Tem temporateEntity.getCreatedAt()); } + /** + * Retrieves temporate content from the temporate repository with the given location, provider and secret. + * + * @param location given temporate content location. + * @param validationSecretsUnit given validation secrets unit. + * @return retrieved temporate content. + */ + public TemporateContentUnitDto retrieveTemporateContentByLocationProviderAndSecret(String location, ValidationSecretsUnit validationSecretsUnit) + throws TemporateContentRemovalFailureException { + ProviderEntity provider; + + try { + provider = providerRepository.findByName(validationSecretsUnit.getProvider().toString()); + } catch (RepositoryOperationFailureException e) { + throw new TemporateContentRemovalFailureException(e.getMessage()); + } + + String signature = repositoryConfigurationHelper.getExternalCredentials( + validationSecretsUnit.getProvider(), + validationSecretsUnit.getCredentials().getExternal()); + + try { + if (!secretRepository.isPresentBySessionAndCredentials( + validationSecretsUnit.getCredentials().getInternal().getId(), signature)) { + throw new TemporateContentRemovalFailureException( + new RepositoryContentApplicationNotExistsException().getMessage()); + } + } catch (RepositoryOperationFailureException e) { + throw new TemporateContentRemovalFailureException(e.getMessage()); + } + + SecretEntity secret; + + try { + secret = secretRepository.findBySessionAndCredentials( + validationSecretsUnit.getCredentials().getInternal().getId(), + signature); + } catch (RepositoryOperationFailureException e) { + throw new TemporateContentRemovalFailureException(e.getMessage()); + } + + TemporateEntity temporate; + + try { + temporate = temporateRepository + .findEarliestByLocationProviderAndSecret(location, provider.getId(), secret.getId()); + } catch (RepositoryOperationFailureException e) { + throw new TemporateContentRemovalFailureException(e.getMessage()); + } + + return TemporateContentUnitDto.of( + temporate.getProvider(), + temporate.getSecret(), + temporate.getLocation(), + temporate.getHash(), + temporate.getCreatedAt()); + } + /** * Retrieves content application from the content repository. * diff --git a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java index cc207d7..2e53437 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java @@ -53,6 +53,10 @@ public void process() throws BackupPeriodRetrievalFailureException { scheduledOperationExecutorService.scheduleWithFixedDelay(() -> { StateService.getBackupProcessorGuard().lock(); +// vendorFacade.listAllObjectsFromBucket( +// +// ); + // vendorFacade.listAllObjectsFromBucket().get(0). // workspaceFacade.add diff --git a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java index 8a7dcd1..f6d3347 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java @@ -12,6 +12,7 @@ import com.objectstorage.service.config.ConfigService; import com.objectstorage.service.state.StateService; import com.objectstorage.service.vendor.VendorFacade; +import com.objectstorage.service.vendor.common.VendorConfigurationHelper; import com.objectstorage.service.workspace.facade.WorkspaceFacade; import io.quarkus.runtime.Startup; import jakarta.annotation.PostConstruct; @@ -156,7 +157,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { vendorFacade.uploadObjectToBucket( contentCompoundUnit.getProvider(), contentCompoundUnit.getCredentialsFieldsFull().getExternal(), - contentCompoundUnit.getRepositoryContentUnitDto().getRoot(), + VendorConfigurationHelper.createBucketName( + contentCompoundUnit.getRepositoryContentUnitDto().getRoot()), temporateContentDto.getLocation(), new ByteArrayInputStream(content)); } catch ( diff --git a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java index f610b44..eedddf3 100644 --- a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java +++ b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java @@ -1,6 +1,7 @@ package com.objectstorage.service.processor; import com.objectstorage.dto.RepositoryContentUnitDto; +import com.objectstorage.dto.TemporateContentUnitDto; import com.objectstorage.exception.*; import com.objectstorage.model.*; import com.objectstorage.repository.executor.RepositoryExecutor; @@ -296,9 +297,19 @@ public byte[] downloadObject( String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); + TemporateContentUnitDto temporateContentUnit; + + try { + temporateContentUnit = + repositoryFacade.retrieveTemporateContentByLocationProviderAndSecret( + location, validationSecretsUnit); + } catch (TemporateContentRemovalFailureException e) { + throw new ProcessorContentDownloadFailureException(e.getMessage()); + } + try { - if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, location)) { - return workspaceFacade.getObjectFile(workspaceUnitKey, location); + if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, temporateContentUnit.getHash())) { + return workspaceFacade.getObjectFile(workspaceUnitKey, temporateContentUnit.getHash()); } } catch (FileExistenceCheckFailureException | FileUnitRetrievalFailureException e) { throw new ProcessorContentDownloadFailureException(e.getMessage()); @@ -453,20 +464,36 @@ public void removeObject( throw new ProcessorContentRemovalFailureException(e1.getMessage()); } - } - try { - if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, location)) { - workspaceFacade.removeObjectFile(workspaceUnitKey, location); - } - } catch (FileExistenceCheckFailureException | FileRemovalFailureException e1) { + TemporateContentUnitDto temporateContentUnit; + try { - repositoryExecutor.rollbackTransaction(); - } catch (TransactionRollbackFailureException e2) { - throw new ProcessorContentRemovalFailureException(e2.getMessage()); + temporateContentUnit = + repositoryFacade.retrieveTemporateContentByLocationProviderAndSecret( + location, validationSecretsUnit); + } catch (TemporateContentRemovalFailureException e1) { + try { + repositoryExecutor.rollbackTransaction(); + } catch (TransactionRollbackFailureException e2) { + throw new ProcessorContentRemovalFailureException(e2.getMessage()); + } + + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } - throw new ProcessorContentRemovalFailureException(e1.getMessage()); + try { + if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, temporateContentUnit.getHash())) { + workspaceFacade.removeObjectFile(workspaceUnitKey, temporateContentUnit.getHash()); + } + } catch (FileExistenceCheckFailureException | FileRemovalFailureException e1) { + try { + repositoryExecutor.rollbackTransaction(); + } catch (TransactionRollbackFailureException e2) { + throw new ProcessorContentRemovalFailureException(e2.getMessage()); + } + + throw new ProcessorContentRemovalFailureException(e1.getMessage()); + } } try { diff --git a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java index 75b7be8..cacc540 100644 --- a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java +++ b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java @@ -67,8 +67,6 @@ private void configure() { } }, 0, properties.getDiagnosticsScrapeDelay(), TimeUnit.MILLISECONDS); - System.out.println(configService.getConfig().getTemporateStorage().getFrequency()); - telemetryBinding.getTemporateStorageFilesAmount().set(10); } diff --git a/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java b/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java index c916537..9316e38 100644 --- a/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java +++ b/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java @@ -172,6 +172,11 @@ public void uploadObjectToS3Bucket( put("objectstorage", "true"); } }); + try { + metadata.setContentLength(inputStream.available()); + } catch (IOException e) { + throw new VendorOperationFailureException(e.getMessage()); + } PutObjectRequest request = new PutObjectRequest(bucketName, fileName, inputStream, metadata); diff --git a/api-server/src/main/openapi/openapi.yml b/api-server/src/main/openapi/openapi.yml index 136b787..b89e1f0 100644 --- a/api-server/src/main/openapi/openapi.yml +++ b/api-server/src/main/openapi/openapi.yml @@ -276,9 +276,13 @@ components: ContentRetrievalProviderUnit: required: - location + - created_at properties: location: type: string + created_at: + type: integer + format: int64 ContentApplication: required: - root From 07e43b6a50de52560ced0ab6eef1b0d05ca576d9 Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi Date: Thu, 28 Nov 2024 13:07:21 +0100 Subject: [PATCH 3/9] fix: fixed bugs --- .../dto/VendorObjectListingDto.java | 21 +++++++++++ .../repository/facade/RepositoryFacade.java | 16 +++++++-- .../integration/backup/BackupService.java | 8 ----- .../service/processor/ProcessorService.java | 35 +++++++++++-------- .../service/telemetry/TelemetryService.java | 20 +++-------- .../service/vendor/VendorFacade.java | 6 ++-- .../service/vendor/gcs/GCSVendorService.java | 7 ++-- .../service/vendor/s3/S3VendorService.java | 6 ++-- 8 files changed, 72 insertions(+), 47 deletions(-) create mode 100644 api-server/src/main/java/com/objectstorage/dto/VendorObjectListingDto.java diff --git a/api-server/src/main/java/com/objectstorage/dto/VendorObjectListingDto.java b/api-server/src/main/java/com/objectstorage/dto/VendorObjectListingDto.java new file mode 100644 index 0000000..25e760c --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/VendorObjectListingDto.java @@ -0,0 +1,21 @@ +package com.objectstorage.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * Represents vendor object listing. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class VendorObjectListingDto { + /** + * Represent location. + */ + private String location; + + /** + * Represents created at timestamp. + */ + private Long createdAt; +} diff --git a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java index 771fa09..6bb351e 100644 --- a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java +++ b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java @@ -12,6 +12,7 @@ import com.objectstorage.model.*; import com.objectstorage.repository.*; import com.objectstorage.repository.common.RepositoryConfigurationHelper; +import com.objectstorage.service.telemetry.TelemetryService; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -44,6 +45,9 @@ public class RepositoryFacade { @Inject SecretRepository secretRepository; + @Inject + TelemetryService telemetryService; + /** * Retrieves filtered content from temporate repository. * @@ -94,11 +98,17 @@ public List retrieveFilteredTemporateContent( * @throws TemporateContentRetrievalFailureException if temporate content amount retrieval fails. */ public Boolean isTemporateContentPresent() throws TemporateContentRetrievalFailureException { + Integer amount; + try { - return temporateRepository.count() > 0; + amount = temporateRepository.count(); } catch (RepositoryOperationFailureException e) { throw new TemporateContentRetrievalFailureException(e.getMessage()); } + + telemetryService.setTemporateStorageFilesAmount(amount); + + return amount > 0; } /** @@ -225,8 +235,8 @@ public TemporateContentUnitDto retrieveTemporateContentByLocationProviderAndSecr try { temporate = temporateRepository .findEarliestByLocationProviderAndSecret(location, provider.getId(), secret.getId()); - } catch (RepositoryOperationFailureException e) { - throw new TemporateContentRemovalFailureException(e.getMessage()); + } catch (RepositoryOperationFailureException ignored) { + return null; } return TemporateContentUnitDto.of( diff --git a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java index 2e53437..ed572ce 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java @@ -53,14 +53,6 @@ public void process() throws BackupPeriodRetrievalFailureException { scheduledOperationExecutorService.scheduleWithFixedDelay(() -> { StateService.getBackupProcessorGuard().lock(); -// vendorFacade.listAllObjectsFromBucket( -// -// ); - -// vendorFacade.listAllObjectsFromBucket().get(0). - -// workspaceFacade.add - StateService.getBackupProcessorGuard().unlock(); }, 0, period, TimeUnit.MILLISECONDS); } diff --git a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java index eedddf3..f3b463b 100644 --- a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java +++ b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java @@ -18,6 +18,7 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * Provides high-level access to ObjectStorage processor operations. @@ -307,12 +308,14 @@ public byte[] downloadObject( throw new ProcessorContentDownloadFailureException(e.getMessage()); } - try { - if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, temporateContentUnit.getHash())) { - return workspaceFacade.getObjectFile(workspaceUnitKey, temporateContentUnit.getHash()); + if (Objects.nonNull(temporateContentUnit)) { + try { + if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, temporateContentUnit.getHash())) { + return workspaceFacade.getObjectFile(workspaceUnitKey, temporateContentUnit.getHash()); + } + } catch (FileExistenceCheckFailureException | FileUnitRetrievalFailureException e) { + throw new ProcessorContentDownloadFailureException(e.getMessage()); } - } catch (FileExistenceCheckFailureException | FileUnitRetrievalFailureException e) { - throw new ProcessorContentDownloadFailureException(e.getMessage()); } RepositoryContentUnitDto repositoryContentLocationUnitDto; @@ -481,18 +484,20 @@ public void removeObject( throw new ProcessorContentRemovalFailureException(e1.getMessage()); } - try { - if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, temporateContentUnit.getHash())) { - workspaceFacade.removeObjectFile(workspaceUnitKey, temporateContentUnit.getHash()); - } - } catch (FileExistenceCheckFailureException | FileRemovalFailureException e1) { + if (Objects.nonNull(temporateContentUnit)) { try { - repositoryExecutor.rollbackTransaction(); - } catch (TransactionRollbackFailureException e2) { - throw new ProcessorContentRemovalFailureException(e2.getMessage()); + if (workspaceFacade.isObjectFilePresent(workspaceUnitKey, temporateContentUnit.getHash())) { + workspaceFacade.removeObjectFile(workspaceUnitKey, temporateContentUnit.getHash()); + } + } catch (FileExistenceCheckFailureException | FileRemovalFailureException e1) { + try { + repositoryExecutor.rollbackTransaction(); + } catch (TransactionRollbackFailureException e2) { + throw new ProcessorContentRemovalFailureException(e2.getMessage()); + } + + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } - - throw new ProcessorContentRemovalFailureException(e1.getMessage()); } } diff --git a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java index cacc540..98881b8 100644 --- a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java +++ b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java @@ -71,24 +71,14 @@ private void configure() { } /** - * Increases current amount of files in ObjectStorage Temporate Storage. + * Sets current amount of files in ObjectStorage Temporate Storage. + * + * @param value given value. */ - public void increaseTemporateStorageFilesAmount() { + public void setTemporateStorageFilesAmount(Integer value) { if (configService.getConfig().getDiagnostics().getEnabled()) { temporateStorageFilesAmountQueue.add( - () -> telemetryBinding.getTemporateStorageFilesAmount().set( - telemetryBinding.getTemporateStorageFilesAmount().get() + 1)); - } - } - - /** - * Decreases current amount of files in ObjectStorage Temporate Storage. - */ - public void decreaseTemporateStorageFilesAmount() { - if (configService.getConfig().getDiagnostics().getEnabled()) { - temporateStorageFilesAmountQueue.add( - () -> telemetryBinding.getTemporateStorageFilesAmount().set( - telemetryBinding.getTemporateStorageFilesAmount().get() - 1)); + () -> telemetryBinding.getTemporateStorageFilesAmount().set(value)); } } diff --git a/api-server/src/main/java/com/objectstorage/service/vendor/VendorFacade.java b/api-server/src/main/java/com/objectstorage/service/vendor/VendorFacade.java index 8751f64..930b273 100644 --- a/api-server/src/main/java/com/objectstorage/service/vendor/VendorFacade.java +++ b/api-server/src/main/java/com/objectstorage/service/vendor/VendorFacade.java @@ -319,7 +319,8 @@ public List listAllObjectsFromBucket( bucketName, credentialsFieldExternal.getRegion()) .stream() - .map(ContentRetrievalProviderUnit::of) + .map(element -> ContentRetrievalProviderUnit.of( + element.getLocation(), element.getCreatedAt())) .toList(); } case GCS -> { @@ -333,7 +334,8 @@ public List listAllObjectsFromBucket( yield gcsVendorService.listObjectsFromGCSBucket(credentials, bucketName) .stream() - .map(ContentRetrievalProviderUnit::of) + .map(element -> ContentRetrievalProviderUnit.of( + element.getLocation(), element.getCreatedAt())) .toList(); } }; diff --git a/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java b/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java index b03d84a..2a5acea 100644 --- a/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java +++ b/api-server/src/main/java/com/objectstorage/service/vendor/gcs/GCSVendorService.java @@ -8,6 +8,7 @@ import com.google.cloud.resourcemanager.ResourceManagerOptions; import com.google.cloud.resourcemanager.Project; import com.google.cloud.storage.*; +import com.objectstorage.dto.VendorObjectListingDto; import com.objectstorage.exception.GCPCredentialsInitializationFailureException; import com.objectstorage.exception.GCSBucketObjectUploadFailureException; import jakarta.enterprise.context.ApplicationScoped; @@ -173,7 +174,7 @@ public byte[] retrieveObjectFromGCSBucket( * @param bucketName given name of the GCS bucket. * @return listed objects. */ - public List listObjectsFromGCSBucket( + public List listObjectsFromGCSBucket( Credentials credentials, String bucketName) { Storage storage = StorageOptions.newBuilder() @@ -184,7 +185,9 @@ public List listObjectsFromGCSBucket( Page blobs = storage.list(bucketName); return StreamSupport.stream(blobs.iterateAll().spliterator(), false) - .map(element -> element.getBlobId().getName()) + .map(element -> VendorObjectListingDto.of( + element.getBlobId().getName(), + element.getUpdateTimeOffsetDateTime().toEpochSecond())) .toList(); } diff --git a/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java b/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java index 9316e38..d8ea071 100644 --- a/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java +++ b/api-server/src/main/java/com/objectstorage/service/vendor/s3/S3VendorService.java @@ -12,6 +12,7 @@ import com.amazonaws.services.securitytoken.model.GetCallerIdentityRequest; import com.amazonaws.waiters.WaiterParameters; import com.objectstorage.dto.AWSSecretsDto; +import com.objectstorage.dto.VendorObjectListingDto; import com.objectstorage.exception.S3BucketObjectRetrievalFailureException; import com.objectstorage.exception.VendorOperationFailureException; import jakarta.enterprise.context.ApplicationScoped; @@ -271,7 +272,7 @@ public byte[] retrieveObjectFromS3Bucket( * @return listed objects. * @throws VendorOperationFailureException if vendor operation fails. */ - public List listObjectsFromS3Bucket( + public List listObjectsFromS3Bucket( AWSCredentialsProvider awsCredentialsProvider, String bucketName, String region) throws VendorOperationFailureException { @@ -285,7 +286,8 @@ public List listObjectsFromS3Bucket( return simpleStorage .listObjects(bucketName) .getObjectSummaries() - .stream().map(S3ObjectSummary::getKey) + .stream().map(element -> VendorObjectListingDto.of( + element.getKey(), element.getLastModified().getTime())) .toList(); } catch (Exception e) { throw new VendorOperationFailureException(e.getMessage()); From 2bfed7ffb19e570cc969bee5a1158c2605ab2f34 Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi Date: Thu, 28 Nov 2024 13:52:53 +0100 Subject: [PATCH 4/9] fix: fixed bugs --- README.md | 2 - .../entity/common/ConfigEntity.java | 28 -- .../TemporateStorageService.java | 66 +++- .../service/state/StateService.java | 7 + .../service/state/watcher/WatcherService.java | 24 ++ .../service/telemetry/TelemetryService.java | 44 +-- .../telemetry/binding/TelemetryBinding.java | 10 +- config/grafana/dashboards/diagnostics.tmpl | 327 +++--------------- samples/config/api-server/api-server.yaml | 5 +- 9 files changed, 133 insertions(+), 380 deletions(-) create mode 100644 api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java diff --git a/README.md b/README.md index 9b9b22c..559538a 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,6 @@ Features: * Provides configurable data backups per vendor in the same workspace * Makes user interaction similar to filesystem -! A mapping of file system to external vendor providers. - ![](./docs/high-level-design.png) ![](./docs/detailed-design.png) diff --git a/api-server/src/main/java/com/objectstorage/entity/common/ConfigEntity.java b/api-server/src/main/java/com/objectstorage/entity/common/ConfigEntity.java index 942eaa3..361f673 100644 --- a/api-server/src/main/java/com/objectstorage/entity/common/ConfigEntity.java +++ b/api-server/src/main/java/com/objectstorage/entity/common/ConfigEntity.java @@ -56,34 +56,6 @@ public static class Security { */ @Getter public static class TemporateStorage { - /** - * Represents all supported content formats, which can be used by ObjectStorage - * Temporate Storage. - */ - @Getter - public enum Format { - @JsonProperty("zip") - ZIP("zip"), - - @JsonProperty("tar") - TAR("tar"); - - private final String value; - - Format(String value) { - this.value = value; - } - - public String toString() { - return value; - } - } - - @Valid - @NotNull - @JsonProperty("format") - public Format format; - @NotNull @JsonProperty("frequency") public String frequency; diff --git a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java index f6d3347..4940513 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java @@ -11,6 +11,8 @@ import com.objectstorage.repository.facade.RepositoryFacade; import com.objectstorage.service.config.ConfigService; import com.objectstorage.service.state.StateService; +import com.objectstorage.service.telemetry.TelemetryService; +import com.objectstorage.service.telemetry.binding.TelemetryBinding; import com.objectstorage.service.vendor.VendorFacade; import com.objectstorage.service.vendor.common.VendorConfigurationHelper; import com.objectstorage.service.workspace.facade.WorkspaceFacade; @@ -50,6 +52,12 @@ public class TemporateStorageService { @Inject VendorFacade vendorFacade; + @Inject + TelemetryBinding telemetryBinding; + + @Inject + TelemetryService telemetryService; + private final ScheduledExecutorService scheduledOperationExecutorService = Executors.newSingleThreadScheduledExecutor(); @@ -69,6 +77,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { throw new TemporateStoragePeriodRetrievalFailureException(e.getMessage()); } + telemetryBinding.getConfiguredTemporateStorageAwaitTime().set(period); + scheduledOperationExecutorService.scheduleWithFixedDelay(() -> { StateService.getTemporateStorageProcessorGuard().lock(); @@ -81,7 +91,9 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { } catch (TemporateContentRetrievalFailureException e1) { StateService.getTemporateStorageProcessorGuard().unlock(); - logger.fatal(e1.getMessage()); + telemetryService.increaseCloudServiceUploadRetries(); + + logger.error(e1.getMessage()); throw new RuntimeException(e1); } @@ -93,7 +105,9 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { } catch (TemporateContentRetrievalFailureException e) { StateService.getTemporateStorageProcessorGuard().unlock(); - logger.fatal(e.getMessage()); + telemetryService.increaseCloudServiceUploadRetries(); + + logger.error(e.getMessage()); throw new RuntimeException(e); } @@ -103,7 +117,9 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { } catch (TransactionInitializationFailureException e) { StateService.getTemporateStorageProcessorGuard().unlock(); - logger.fatal(e.getMessage()); + telemetryService.increaseCloudServiceUploadRetries(); + + logger.error(e.getMessage()); throw new RuntimeException(e); } @@ -111,17 +127,21 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { try { repositoryFacade.removeTemporateContentByHash(temporateContentDto.getHash()); } catch (TemporateContentRemovalFailureException e1) { - StateService.getTemporateStorageProcessorGuard().unlock(); + telemetryService.increaseCloudServiceUploadRetries(); try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { - logger.fatal(e2.getMessage()); + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e2.getMessage()); throw new RuntimeException(e2); } - logger.fatal(e1.getMessage()); + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e1.getMessage()); throw new RuntimeException(e1); } @@ -137,17 +157,21 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { try { content = workspaceFacade.getObjectFile(workspaceUnitKey, temporateContentDto.getHash()); } catch (FileUnitRetrievalFailureException e1) { - StateService.getTemporateStorageProcessorGuard().unlock(); + telemetryService.increaseCloudServiceUploadRetries(); try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { - logger.fatal(e2.getMessage()); + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e2.getMessage()); throw new RuntimeException(e2); } - logger.fatal(e1.getMessage()); + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e1.getMessage()); throw new RuntimeException(e1); } @@ -165,16 +189,20 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { SecretsConversionException | VendorOperationFailureException | BucketObjectUploadFailureException e1) { - StateService.getTemporateStorageProcessorGuard().unlock(); + telemetryService.increaseCloudServiceUploadRetries(); try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { - logger.fatal(e2.getMessage()); + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e2.getMessage()); throw new RuntimeException(e2); } + StateService.getTemporateStorageProcessorGuard().unlock(); + logger.info(e1.getMessage()); return; @@ -184,17 +212,21 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { try { workspaceFacade.removeObjectFile(workspaceUnitKey, temporateContentDto.getHash()); } catch (FileRemovalFailureException e1) { - StateService.getTemporateStorageProcessorGuard().unlock(); + telemetryService.increaseCloudServiceUploadRetries(); try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { - logger.fatal(e2.getMessage()); + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e2.getMessage()); throw new RuntimeException(e2); } - logger.info(e1.getMessage()); + StateService.getTemporateStorageProcessorGuard().unlock(); + + logger.error(e1.getMessage()); } try { @@ -202,11 +234,15 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { } catch (TransactionCommitFailureException e) { StateService.getTemporateStorageProcessorGuard().unlock(); - logger.fatal(e.getMessage()); + telemetryService.increaseCloudServiceUploadRetries(); + + logger.error(e.getMessage()); throw new RuntimeException(e); } + telemetryService.increaseCurrentCloudServiceUploads(); + StateService.getTemporateStorageProcessorGuard().unlock(); }, 0, period, TimeUnit.MILLISECONDS); } diff --git a/api-server/src/main/java/com/objectstorage/service/state/StateService.java b/api-server/src/main/java/com/objectstorage/service/state/StateService.java index b2317b1..cccf84c 100644 --- a/api-server/src/main/java/com/objectstorage/service/state/StateService.java +++ b/api-server/src/main/java/com/objectstorage/service/state/StateService.java @@ -1,5 +1,6 @@ package com.objectstorage.service.state; +import com.objectstorage.service.state.watcher.WatcherService; import lombok.Getter; import lombok.Setter; @@ -27,4 +28,10 @@ public class StateService { */ @Getter private final static ReentrantLock backupProcessorGuard = new ReentrantLock(); + + /** + * Represents ObjectStorage watcher service instance. + */ + @Getter + private static WatcherService watcherService; } diff --git a/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java b/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java new file mode 100644 index 0000000..8fccf71 --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java @@ -0,0 +1,24 @@ +package com.objectstorage.service.state.watcher; + +import lombok.Getter; +import lombok.Setter; + +/** + * Service used to track state metrics for the current session in the application. + */ +public class WatcherService { + + /** + * Represents amount of files uploaded to ObjectStorage Temporate Storage in the current session. + */ + @Getter + @Setter + private static Double filesUploadCounter = (double) 0; + + /** + * Represents global amount of files content uploaded in the current session. + */ + @Getter + @Setter + private static Double uploadedFilesSize = (double) 0; +} diff --git a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java index 98881b8..be88cf3 100644 --- a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java +++ b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java @@ -66,8 +66,6 @@ private void configure() { averageUploadFileSizeQueue.poll().run(); } }, 0, properties.getDiagnosticsScrapeDelay(), TimeUnit.MILLISECONDS); - - telemetryBinding.getTemporateStorageFilesAmount().set(10); } /** @@ -93,17 +91,6 @@ public void increaseCurrentCloudServiceUploads() { } } - /** - * Decreases current cloud service uploads from ObjectStorage Temporate Storage. - */ - public void decreaseCurrentCloudServiceUploads() { - if (configService.getConfig().getDiagnostics().getEnabled()) { - currentCloudServiceUploadsQueue.add( - () -> telemetryBinding.getCurrentCloudServiceUploadsAmount().set( - telemetryBinding.getCurrentCloudServiceUploadsAmount().get() - 1)); - } - } - /** * Increases cloud service upload retries form ObjectStorage Temporate Storage. */ @@ -116,35 +103,14 @@ public void increaseCloudServiceUploadRetries() { } /** - * Decreases cloud service upload retries form ObjectStorage Temporate Storage. - */ - public void decreaseCloudServiceUploadRetries() { - if (configService.getConfig().getDiagnostics().getEnabled()) { - cloudServiceUploadRetriesQueue.add( - () -> telemetryBinding.getCloudServiceUploadRetries().set( - telemetryBinding.getCloudServiceUploadRetries().get() - 1)); - } - } - - /** - * Increases average upload file size to ObjectStorage Temporate Storage. - */ - public void increaseAverageUploadFileSizeQueue() { - if (configService.getConfig().getDiagnostics().getEnabled()) { - averageUploadFileSizeQueue.add( - () -> telemetryBinding.getAverageUploadFileSize().set( - telemetryBinding.getAverageUploadFileSize().get() + 1)); - } - } - - /** - * Decreases average upload file size to ObjectStorage Temporate Storage. + * Sets average upload file size to ObjectStorage Temporate Storage. + * + * @param value given value. */ - public void decreaseAverageUploadFileSizeQueue() { + public void setAverageUploadFileSizeQueue(Double value) { if (configService.getConfig().getDiagnostics().getEnabled()) { averageUploadFileSizeQueue.add( - () -> telemetryBinding.getAverageUploadFileSize().set( - telemetryBinding.getAverageUploadFileSize().get() - 1)); + () -> telemetryBinding.getAverageUploadFileSize().set(value)); } } } diff --git a/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java b/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java index 79957ad..6c6b8f4 100644 --- a/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java +++ b/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java @@ -1,5 +1,6 @@ package com.objectstorage.service.telemetry.binding; +import com.google.common.util.concurrent.AtomicDouble; import io.micrometer.core.instrument.Timer; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; @@ -9,6 +10,7 @@ import org.jetbrains.annotations.NotNull; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; /** * Service used to create custom telemetry bindings used to distribute application metrics. @@ -22,9 +24,9 @@ public class TelemetryBinding implements MeterBinder { private final AtomicInteger cloudServiceUploadRetries = new AtomicInteger(); - private final AtomicInteger configuredTemporateStorageAwaitTime = new AtomicInteger(); + private final AtomicLong configuredTemporateStorageAwaitTime = new AtomicLong(); - private final AtomicInteger averageUploadFileSize = new AtomicInteger(); + private final AtomicDouble averageUploadFileSize = new AtomicDouble(); /** * @see MeterBinder @@ -43,11 +45,11 @@ public void bindTo(@NotNull MeterRegistry meterRegistry) { .description("Represents cloud service uploads retries from ObjectStorage Temporate Storage") .register(meterRegistry); - Gauge.builder("general.configured_temporate_storage_await_time", configuredTemporateStorageAwaitTime, AtomicInteger::get) + Gauge.builder("general.configured_temporate_storage_await_time", configuredTemporateStorageAwaitTime, AtomicLong::get) .description("Represents configured ObjectStorage Temporate Storage await time") .register(meterRegistry); - Gauge.builder("general.average_upload_file_size", averageUploadFileSize, AtomicInteger::get) + Gauge.builder("general.average_upload_file_size", averageUploadFileSize, AtomicDouble::get) .description("Represents average upload file size in ObjectStorage Temporate Storage") .register(meterRegistry); } diff --git a/config/grafana/dashboards/diagnostics.tmpl b/config/grafana/dashboards/diagnostics.tmpl index 8d6ee04..94e1fa7 100644 --- a/config/grafana/dashboards/diagnostics.tmpl +++ b/config/grafana/dashboards/diagnostics.tmpl @@ -20,7 +20,6 @@ "fiscalYearStartMonth": 0, "gnetId": 179, "graphTooltip": 1, - "id": 1, "links": [], "panels": [ { @@ -31,120 +30,6 @@ "x": 0, "y": 0 }, - "id": 30, - "panels": [], - "title": "ObjectStorage Cluster", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P21B111CBFE6E8FCA" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 38, - "interval": "500ms", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P21B111CBFE6E8FCA" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "general_cluster_download_amount", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "ObjectStorage Cluster content download ongoing requests", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, "id": 29, "panels": [], "title": "ObjectStorage API Server", @@ -214,7 +99,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 1 }, "id": 36, "interval": "500ms", @@ -238,7 +123,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "general_raw_content_upload_amount", + "expr": "general_temporate_storage_files_amount", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -248,7 +133,7 @@ "useBackend": false } ], - "title": "ObjectStorage API Server raw content upload ongoing requests", + "title": "ObjectStorage API Server Temporate Storage current files amount", "type": "timeseries" }, { @@ -315,7 +200,7 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 1 }, "id": 37, "interval": "500ms", @@ -339,7 +224,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "general_additional_content_upload_amount", + "expr": "general_current_cloud_service_uploads_amount", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -349,7 +234,7 @@ "useBackend": false } ], - "title": "ObjectStorage API Server additional content upload ongoing requests", + "title": "ObjectStorage API Server current cloud service uploads amount", "type": "timeseries" }, { @@ -416,7 +301,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 9 }, "id": 33, "interval": "0.1", @@ -440,7 +325,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "general_api_server_health_check_amount", + "expr": "general_cloud_service_upload_retries", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -450,7 +335,7 @@ "useBackend": false } ], - "title": "ObjectStorage API Server healthcheck ongoing requests", + "title": "ObjectStorage API Server cloud service upload retries", "type": "timeseries" }, { @@ -517,7 +402,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 9 }, "id": 34, "interval": "0.1", @@ -541,7 +426,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "general_cluster_health_check_amount", + "expr": "general_average_upload_file_size", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -551,7 +436,7 @@ "useBackend": false } ], - "title": "ObjectStorage Cluster healthcheck requests", + "title": "ObjectStorage API Server average upload file size", "type": "timeseries" }, { @@ -561,41 +446,6 @@ }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepBefore", - "lineWidth": 1, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, "decimals": 0, "fieldMinMax": false, "mappings": [], @@ -605,137 +455,38 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 31, - "interval": "0.1", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P21B111CBFE6E8FCA" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "general_serving_cluster_amount", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Serving ObjectStorage Cluster allocations", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P21B111CBFE6E8FCA" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepBefore", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 26 + "x": 0, + "y": 17 }, "id": 32, "interval": "0.1", "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -744,7 +495,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "general_suspended_cluster_amount", + "expr": "general_configured_temporate_storage_await_time", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -754,8 +505,8 @@ "useBackend": false } ], - "title": "Suspended ObjectStorage Cluster allocations", - "type": "timeseries" + "title": "ObjectStorage API Server Temporate Storage confiugred await time(milliseconds)", + "type": "stat" }, { "datasource": { @@ -786,8 +537,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 34 + "x": 12, + "y": 17 }, "id": 35, "options": { @@ -835,7 +586,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 25 }, "id": 28, "panels": [], @@ -884,7 +635,7 @@ "h": 7, "w": 3, "x": 0, - "y": 43 + "y": 26 }, "id": 15, "maxDataPoints": 100, @@ -969,7 +720,7 @@ "h": 6, "w": 6, "x": 3, - "y": 43 + "y": 26 }, "id": 6, "maxDataPoints": 100, @@ -1055,7 +806,7 @@ "h": 6, "w": 6, "x": 9, - "y": 43 + "y": 26 }, "id": 4, "maxDataPoints": 100, @@ -1300,6 +1051,6 @@ "timezone": "browser", "title": "ObjectStorage Diagnostics", "uid": "64nrElFmk", - "version": 10, + "version": 3, "weekStart": "" } \ No newline at end of file diff --git a/samples/config/api-server/api-server.yaml b/samples/config/api-server/api-server.yaml index f1f2c63..4022b0a 100644 --- a/samples/config/api-server/api-server.yaml +++ b/samples/config/api-server/api-server.yaml @@ -17,9 +17,6 @@ connection: # Represents section used for ObjectStorage API Server temporate storage configuration. Same compression will be # used to upload files to the configured cloud providers. temporate-storage: - # Represents format used for content to be saved. - format: "zip" - # Represents frequency of scheduled operations processing. frequency: "*/5 * * * * ?" @@ -37,7 +34,7 @@ backup: # Represents section used for ObjectStorage API Server diagnostics configuration. diagnostics: # Enables diagnostics functionality. - enabled: false + enabled: true # Represents section used for ObjectStorage diagnostics metrics configuration. metrics: From 06cbf1f43f9abed595c823362066fcb7f8630efe Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi Date: Thu, 28 Nov 2024 14:06:30 +0100 Subject: [PATCH 5/9] fix: fixed bugs --- .../TemporateStorageService.java | 22 ++++++++++--------- config/grafana/dashboards/diagnostics.tmpl | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java index 4940513..b923f00 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java @@ -95,7 +95,7 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { logger.error(e1.getMessage()); - throw new RuntimeException(e1); + return; } EarliestTemporateContentDto temporateContentDto; @@ -109,7 +109,7 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { logger.error(e.getMessage()); - throw new RuntimeException(e); + return; } try { @@ -121,7 +121,7 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { logger.error(e.getMessage()); - throw new RuntimeException(e); + return; } try { @@ -136,14 +136,14 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { logger.error(e2.getMessage()); - throw new RuntimeException(e2); + return; } StateService.getTemporateStorageProcessorGuard().unlock(); logger.error(e1.getMessage()); - throw new RuntimeException(e1); + return; } ValidationSecretsApplication validationSecretsApplication = @@ -166,14 +166,14 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { logger.error(e2.getMessage()); - throw new RuntimeException(e2); + return; } StateService.getTemporateStorageProcessorGuard().unlock(); logger.error(e1.getMessage()); - throw new RuntimeException(e1); + return; } for (ContentCompoundUnitDto contentCompoundUnit : temporateContentDto.getContentCompoundUnits()) { @@ -198,7 +198,7 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { logger.error(e2.getMessage()); - throw new RuntimeException(e2); + return; } StateService.getTemporateStorageProcessorGuard().unlock(); @@ -221,12 +221,14 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { logger.error(e2.getMessage()); - throw new RuntimeException(e2); + return; } StateService.getTemporateStorageProcessorGuard().unlock(); logger.error(e1.getMessage()); + + return; } try { @@ -238,7 +240,7 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { logger.error(e.getMessage()); - throw new RuntimeException(e); + return; } telemetryService.increaseCurrentCloudServiceUploads(); diff --git a/config/grafana/dashboards/diagnostics.tmpl b/config/grafana/dashboards/diagnostics.tmpl index 94e1fa7..cd6362f 100644 --- a/config/grafana/dashboards/diagnostics.tmpl +++ b/config/grafana/dashboards/diagnostics.tmpl @@ -505,7 +505,7 @@ "useBackend": false } ], - "title": "ObjectStorage API Server Temporate Storage confiugred await time(milliseconds)", + "title": "ObjectStorage API Server Temporate Storage configured await time(milliseconds)", "type": "stat" }, { From e636884bf85ea22e2cac74325f81309a95d327ba Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi Date: Fri, 29 Nov 2024 03:23:09 +0100 Subject: [PATCH 6/9] fix: fixed bugs --- .../TemporateStorageService.java | 24 ++++++ .../service/processor/ProcessorService.java | 84 +++++++++++++++++++ .../service/state/StateService.java | 6 ++ 3 files changed, 114 insertions(+) diff --git a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java index b923f00..57d4807 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java @@ -112,9 +112,13 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { return; } + StateService.getTransactionProcessorGuard().lock(); + try { repositoryExecutor.beginTransaction(); } catch (TransactionInitializationFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); telemetryService.increaseCloudServiceUploadRetries(); @@ -132,6 +136,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); logger.error(e2.getMessage()); @@ -139,6 +145,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { return; } + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); logger.error(e1.getMessage()); @@ -162,6 +170,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); logger.error(e2.getMessage()); @@ -169,6 +179,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { return; } + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); logger.error(e1.getMessage()); @@ -194,6 +206,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); logger.error(e2.getMessage()); @@ -201,6 +215,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { return; } + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); logger.info(e1.getMessage()); @@ -217,6 +233,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); logger.error(e2.getMessage()); @@ -224,6 +242,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { return; } + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); logger.error(e1.getMessage()); @@ -234,6 +254,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { try { repositoryExecutor.commitTransaction(); } catch (TransactionCommitFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + StateService.getTemporateStorageProcessorGuard().unlock(); telemetryService.increaseCloudServiceUploadRetries(); @@ -243,6 +265,8 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { return; } + StateService.getTransactionProcessorGuard().unlock(); + telemetryService.increaseCurrentCloudServiceUploads(); StateService.getTemporateStorageProcessorGuard().unlock(); diff --git a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java index f3b463b..6f7457f 100644 --- a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java +++ b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java @@ -6,6 +6,7 @@ import com.objectstorage.model.*; import com.objectstorage.repository.executor.RepositoryExecutor; import com.objectstorage.repository.facade.RepositoryFacade; +import com.objectstorage.service.state.StateService; import com.objectstorage.service.telemetry.TelemetryService; import com.objectstorage.service.vendor.VendorFacade; import com.objectstorage.service.vendor.common.VendorConfigurationHelper; @@ -101,9 +102,13 @@ public ContentRetrievalResult retrieveContent(ValidationSecretsApplication valid */ public void apply(ContentApplication contentApplication, ValidationSecretsApplication validationSecretsApplication) throws ProcessorContentApplicationFailureException { + StateService.getTransactionProcessorGuard().lock(); + try { repositoryExecutor.beginTransaction(); } catch (TransactionInitializationFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e.getMessage()); } @@ -114,9 +119,13 @@ public void apply(ContentApplication contentApplication, ValidationSecretsApplic try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e1.getMessage()); } @@ -136,9 +145,13 @@ public void apply(ContentApplication contentApplication, ValidationSecretsApplic try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e1.getMessage()); } } @@ -146,8 +159,12 @@ public void apply(ContentApplication contentApplication, ValidationSecretsApplic try { repositoryExecutor.commitTransaction(); } catch (TransactionCommitFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentApplicationFailureException(e.getMessage()); } + + StateService.getTransactionProcessorGuard().unlock(); } /** @@ -159,9 +176,13 @@ public void apply(ContentApplication contentApplication, ValidationSecretsApplic */ public void withdraw(ValidationSecretsApplication validationSecretsApplication) throws ProcessorContentWithdrawalFailureException { + StateService.getTransactionProcessorGuard().lock(); + try { repositoryExecutor.beginTransaction(); } catch (TransactionInitializationFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentWithdrawalFailureException(e.getMessage()); } @@ -174,9 +195,13 @@ public void withdraw(ValidationSecretsApplication validationSecretsApplication) try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentWithdrawalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); } @@ -186,9 +211,13 @@ public void withdraw(ValidationSecretsApplication validationSecretsApplication) try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentWithdrawalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); } @@ -208,9 +237,13 @@ public void withdraw(ValidationSecretsApplication validationSecretsApplication) try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentWithdrawalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); } } @@ -218,8 +251,12 @@ public void withdraw(ValidationSecretsApplication validationSecretsApplication) try { repositoryExecutor.commitTransaction(); } catch (TransactionCommitFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentWithdrawalFailureException(e.getMessage()); } + + StateService.getTransactionProcessorGuard().unlock(); } /** @@ -235,9 +272,13 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp throws ProcessorContentUploadFailureException { logger.info(String.format("Uploading content at '%s' location", location)); + StateService.getTransactionProcessorGuard().lock(); + try { repositoryExecutor.beginTransaction(); } catch (TransactionInitializationFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentUploadFailureException(e.getMessage()); } @@ -253,9 +294,13 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentUploadFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentUploadFailureException(e1.getMessage()); } } @@ -266,8 +311,11 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentUploadFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); throw new ProcessorContentUploadFailureException(e1.getMessage()); } @@ -275,8 +323,12 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp try { repositoryExecutor.commitTransaction(); } catch (TransactionCommitFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentUploadFailureException(e.getMessage()); } + + StateService.getTransactionProcessorGuard().unlock(); } /** @@ -397,9 +449,13 @@ public void removeObject( throws ProcessorContentRemovalFailureException { logger.info(String.format("Removing content object of '%s' location", location)); + StateService.getTransactionProcessorGuard().lock(); + try { repositoryExecutor.beginTransaction(); } catch (TransactionInitializationFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e.getMessage()); } @@ -412,9 +468,13 @@ public void removeObject( try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } @@ -426,9 +486,13 @@ public void removeObject( try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } @@ -445,9 +509,13 @@ public void removeObject( try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } @@ -462,9 +530,13 @@ public void removeObject( try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } @@ -478,9 +550,13 @@ public void removeObject( try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } @@ -493,9 +569,13 @@ public void removeObject( try { repositoryExecutor.rollbackTransaction(); } catch (TransactionRollbackFailureException e2) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e2.getMessage()); } + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e1.getMessage()); } } @@ -504,8 +584,12 @@ public void removeObject( try { repositoryExecutor.commitTransaction(); } catch (TransactionCommitFailureException e) { + StateService.getTransactionProcessorGuard().unlock(); + throw new ProcessorContentRemovalFailureException(e.getMessage()); } + + StateService.getTransactionProcessorGuard().unlock(); } /** diff --git a/api-server/src/main/java/com/objectstorage/service/state/StateService.java b/api-server/src/main/java/com/objectstorage/service/state/StateService.java index cccf84c..c0ef5ff 100644 --- a/api-server/src/main/java/com/objectstorage/service/state/StateService.java +++ b/api-server/src/main/java/com/objectstorage/service/state/StateService.java @@ -29,6 +29,12 @@ public class StateService { @Getter private final static ReentrantLock backupProcessorGuard = new ReentrantLock(); + /** + * Represents ObjectStorage transaction guard. + */ + @Getter + private final static ReentrantLock transactionProcessorGuard = new ReentrantLock(); + /** * Represents ObjectStorage watcher service instance. */ From 426000153e938a37e19bdaf0ce6a60b8ca4fb692 Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi Date: Fri, 29 Nov 2024 03:57:42 +0100 Subject: [PATCH 7/9] fix: fixed bugs --- .../service/processor/ProcessorService.java | 18 ++++++++ .../service/state/StateService.java | 2 +- .../service/state/watcher/WatcherService.java | 43 ++++++++++++++----- config/grafana/dashboards/diagnostics.tmpl | 5 ++- 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java index 6f7457f..0afea18 100644 --- a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java +++ b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java @@ -16,6 +16,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; @@ -282,6 +283,16 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp throw new ProcessorContentUploadFailureException(e.getMessage()); } + Integer fileSize; + + try { + fileSize = file.available(); + } catch (IOException e) { + StateService.getTransactionProcessorGuard().unlock(); + + throw new ProcessorContentUploadFailureException(e.getMessage()); + } + String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); @@ -329,6 +340,13 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp } StateService.getTransactionProcessorGuard().unlock(); + + StateService.getWatcherService().increaseFilesUploadCounter(); + + StateService.getWatcherService().increaseUploadedFilesSize(fileSize); + + telemetryService.setAverageUploadFileSizeQueue( + StateService.getWatcherService().getAverageFileSize()); } /** diff --git a/api-server/src/main/java/com/objectstorage/service/state/StateService.java b/api-server/src/main/java/com/objectstorage/service/state/StateService.java index c0ef5ff..40214b8 100644 --- a/api-server/src/main/java/com/objectstorage/service/state/StateService.java +++ b/api-server/src/main/java/com/objectstorage/service/state/StateService.java @@ -39,5 +39,5 @@ public class StateService { * Represents ObjectStorage watcher service instance. */ @Getter - private static WatcherService watcherService; + private static final WatcherService watcherService = new WatcherService(); } diff --git a/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java b/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java index 8fccf71..bf3a21e 100644 --- a/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java +++ b/api-server/src/main/java/com/objectstorage/service/state/watcher/WatcherService.java @@ -1,24 +1,47 @@ package com.objectstorage.service.state.watcher; -import lombok.Getter; -import lombok.Setter; +import org.apache.commons.io.FileUtils; /** * Service used to track state metrics for the current session in the application. */ public class WatcherService { - /** * Represents amount of files uploaded to ObjectStorage Temporate Storage in the current session. */ - @Getter - @Setter - private static Double filesUploadCounter = (double) 0; + private Integer filesUploadCounter = 0; + + /** + * Increases amount of files uploaded to ObjectStorage Temporate Storage in the current session. + */ + public void increaseFilesUploadCounter() { + filesUploadCounter++; + } /** - * Represents global amount of files content uploaded in the current session. + * Represents global files size uploaded in the current session. */ - @Getter - @Setter - private static Double uploadedFilesSize = (double) 0; + private Integer uploadedFilesSize = 0; + + /** + * Increases global files size uploaded in the current session with the given value. + * + * @param value given value of uploaded file size. + */ + public void increaseUploadedFilesSize(Integer value) { + uploadedFilesSize += value; + } + + /** + * Calculates average file size in the current session. + * + * @return calculated average file size. + */ + public Double getAverageFileSize() { + if (filesUploadCounter > 0) { + return Double.valueOf(uploadedFilesSize) / Double.valueOf(filesUploadCounter) / 1024 / 1024; + } + + return (double) 0; + } } diff --git a/config/grafana/dashboards/diagnostics.tmpl b/config/grafana/dashboards/diagnostics.tmpl index cd6362f..44d81a0 100644 --- a/config/grafana/dashboards/diagnostics.tmpl +++ b/config/grafana/dashboards/diagnostics.tmpl @@ -394,7 +394,8 @@ "value": 80 } ] - } + }, + "unit": "decmbytes" }, "overrides": [] }, @@ -1051,6 +1052,6 @@ "timezone": "browser", "title": "ObjectStorage Diagnostics", "uid": "64nrElFmk", - "version": 3, + "version": 7, "weekStart": "" } \ No newline at end of file From 4e12265087f85b579a44ae5d66633c2d9a80492f Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi Date: Fri, 29 Nov 2024 13:56:57 +0100 Subject: [PATCH 8/9] fix: fixed bugs --- ...alidationSecretsApplicationConverter.java} | 4 +- ...ValidationSecretsApplicationConverter.java | 34 ++++++ .../dto/ContentCompoundUnitDto.java | 2 +- .../dto/FolderContentUnitDto.java | 23 ++++ .../RepositoryContentApplicationUnitDto.java | 28 +++++ .../entity/common/PropertiesEntity.java | 3 + .../repository/ContentRepository.java | 49 ++++++++ .../repository/facade/RepositoryFacade.java | 61 +++++++++- .../integration/backup/BackupService.java | 109 +++++++++++++++++- .../TemporateStorageService.java | 7 +- .../service/processor/ProcessorService.java | 44 ------- .../service/telemetry/TelemetryService.java | 19 +++ .../telemetry/binding/TelemetryBinding.java | 8 +- .../service/workspace/WorkspaceService.java | 39 +++++++ .../common/WorkspaceConfigurationHelper.java | 17 +++ .../workspace/facade/WorkspaceFacade.java | 40 ++++--- api-server/src/main/openapi/openapi.yml | 1 + .../src/main/resources/application.properties | 3 + 18 files changed, 416 insertions(+), 75 deletions(-) rename api-server/src/main/java/com/objectstorage/converter/{ContentCompoundUnitToValidationSecretsApplicationConverter.java => ContentCompoundUnitsToValidationSecretsApplicationConverter.java} (87%) create mode 100644 api-server/src/main/java/com/objectstorage/converter/RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter.java create mode 100644 api-server/src/main/java/com/objectstorage/dto/FolderContentUnitDto.java create mode 100644 api-server/src/main/java/com/objectstorage/dto/RepositoryContentApplicationUnitDto.java create mode 100644 api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java diff --git a/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitToValidationSecretsApplicationConverter.java b/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitsToValidationSecretsApplicationConverter.java similarity index 87% rename from api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitToValidationSecretsApplicationConverter.java rename to api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitsToValidationSecretsApplicationConverter.java index 84ec9c2..180c75b 100644 --- a/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitToValidationSecretsApplicationConverter.java +++ b/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitsToValidationSecretsApplicationConverter.java @@ -10,7 +10,7 @@ /** * Represents content compounds units to validation secrets application converter; */ -public class ContentCompoundUnitToValidationSecretsApplicationConverter { +public class ContentCompoundUnitsToValidationSecretsApplicationConverter { /** * Converts given content compound units to validation secrets application. @@ -25,7 +25,7 @@ public static ValidationSecretsApplication convert(List element -> validationSecretsUnits.add( ValidationSecretsUnit.of( element.getProvider(), - element.getCredentialsFieldsFull()))); + element.getCredentials()))); return ValidationSecretsApplication.of(validationSecretsUnits); } diff --git a/api-server/src/main/java/com/objectstorage/converter/RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter.java b/api-server/src/main/java/com/objectstorage/converter/RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter.java new file mode 100644 index 0000000..8826ad1 --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/converter/RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter.java @@ -0,0 +1,34 @@ +package com.objectstorage.converter; + +import com.objectstorage.dto.ContentCompoundUnitDto; +import com.objectstorage.dto.RepositoryContentApplicationUnitDto; +import com.objectstorage.model.ValidationSecretsApplication; +import com.objectstorage.model.ValidationSecretsUnit; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents repository content application units to validation secrets application converter; + */ +public class RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter { + + /** + * Converts given content compound units to validation secrets application. + * + * @param repositoryContentApplicationUnits given repository content application units. + * @return converted validation secrets application. + */ + public static ValidationSecretsApplication convert( + List repositoryContentApplicationUnits) { + List validationSecretsUnits = new ArrayList<>(); + + repositoryContentApplicationUnits.forEach( + element -> validationSecretsUnits.add( + ValidationSecretsUnit.of( + element.getProvider(), + element.getCredentials()))); + + return ValidationSecretsApplication.of(validationSecretsUnits); + } +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java index 6a944ac..d4643f6 100644 --- a/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java +++ b/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java @@ -25,5 +25,5 @@ public class ContentCompoundUnitDto { /** * Represents full credentials fields. */ - private CredentialsFieldsFull credentialsFieldsFull; + private CredentialsFieldsFull credentials; } diff --git a/api-server/src/main/java/com/objectstorage/dto/FolderContentUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/FolderContentUnitDto.java new file mode 100644 index 0000000..691f727 --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/FolderContentUnitDto.java @@ -0,0 +1,23 @@ +package com.objectstorage.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.io.InputStream; + +/** + * Represents folder content unit. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class FolderContentUnitDto { + /** + * Represents folder entity name. + */ + private String location; + + /** + * Represents folder entity content. + */ + private byte[] content; +} diff --git a/api-server/src/main/java/com/objectstorage/dto/RepositoryContentApplicationUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/RepositoryContentApplicationUnitDto.java new file mode 100644 index 0000000..2750cac --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/RepositoryContentApplicationUnitDto.java @@ -0,0 +1,28 @@ +package com.objectstorage.dto; + +import com.objectstorage.model.CredentialsFieldsFull; +import com.objectstorage.model.Provider; +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * Represents repository content application unit. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class RepositoryContentApplicationUnitDto { + /** + * Represents root location for internal file system. + */ + private String root; + + /** + * Represents provider. + */ + private Provider provider; + + /** + * Represents full credentials fields. + */ + private CredentialsFieldsFull credentials; +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java b/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java index 4cf79a1..ec3e7a9 100644 --- a/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java +++ b/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java @@ -68,6 +68,9 @@ public class PropertiesEntity { @ConfigProperty(name = "workspace.content.backup.directory") String workspaceContentBackupDirectory; + @ConfigProperty(name = "workspace.content.backup.unit") + String workspaceContentBackupUnit; + @ConfigProperty(name = "workspace.compression.file.name") String workspaceCompressionFileName; diff --git a/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java b/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java index 8d4fda1..32412d8 100644 --- a/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java +++ b/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java @@ -102,6 +102,55 @@ public ContentEntity findByProviderAndSecret(Integer provider, Integer secret) t return ContentEntity.of(id, provider, secret, root); } + /** + * Retrieves all the persisted content entities. + * + * @return retrieved content entities. + * @throws RepositoryOperationFailureException if repository operation fails. + */ + public List findAll() throws RepositoryOperationFailureException { + ResultSet resultSet; + + try { + resultSet = + repositoryExecutor.performQueryWithResult( + String.format( + "SELECT t.id, t.root, t.provider, t.secret FROM %s as t", + properties.getDatabaseContentTableName())); + + } catch (QueryExecutionFailureException | QueryEmptyResultException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + List result = new ArrayList<>(); + + Integer id; + String root; + Integer provider; + Integer secret; + + try { + while (resultSet.next()) { + id = resultSet.getInt("id"); + root = resultSet.getString("root"); + provider = resultSet.getInt("provider"); + secret = resultSet.getInt("secret"); + + result.add(ContentEntity.of(id, provider, secret, root)); + } + } catch (SQLException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + try { + resultSet.close(); + } catch (SQLException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + return result; + } + /** * Deletes all entities with the given provider and secret from content table. * diff --git a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java index 6bb351e..370a41e 100644 --- a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java +++ b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java @@ -1,9 +1,6 @@ package com.objectstorage.repository.facade; -import com.objectstorage.dto.ContentCompoundUnitDto; -import com.objectstorage.dto.RepositoryContentUnitDto; -import com.objectstorage.dto.EarliestTemporateContentDto; -import com.objectstorage.dto.TemporateContentUnitDto; +import com.objectstorage.dto.*; import com.objectstorage.entity.repository.ContentEntity; import com.objectstorage.entity.repository.ProviderEntity; import com.objectstorage.entity.repository.SecretEntity; @@ -160,7 +157,7 @@ public EarliestTemporateContentDto retrieveEarliestTemporateContent() throws Tem throw new TemporateContentRetrievalFailureException(e.getMessage()); } - CredentialsFieldsFull secrets = + CredentialsFieldsFull credentials = repositoryConfigurationHelper.convertRawSecretsToContentCredentials( provider, secret.getSession(), @@ -179,7 +176,7 @@ public EarliestTemporateContentDto retrieveEarliestTemporateContent() throws Tem RepositoryContentUnitDto.of( contentEntity.getRoot()), provider, - secrets)); + credentials)); } return EarliestTemporateContentDto.of( @@ -300,6 +297,58 @@ public RepositoryContentUnitDto retrieveContentApplication(ValidationSecretsUnit return RepositoryContentUnitDto.of(contentEntity.getRoot()); } + /** + * Retrieves all content applications from the content repository. + * + * @return retrieved all content applications. + * @throws ContentApplicationRetrievalFailureException if content applications retrieval fails. + */ + public List retrieveAllContentApplications() + throws ContentApplicationRetrievalFailureException { + List contentEntities; + + try { + contentEntities = contentRepository.findAll(); + } catch (RepositoryOperationFailureException e) { + throw new ContentApplicationRetrievalFailureException(e.getMessage()); + } + + List repositoryContentApplicationUnits = new ArrayList<>(); + + for (ContentEntity content : contentEntities) { + ProviderEntity rawProvider; + + try { + rawProvider = providerRepository.findById(content.getProvider()); + } catch (RepositoryOperationFailureException e) { + throw new ContentApplicationRetrievalFailureException(e.getMessage()); + } + + Provider provider = + repositoryConfigurationHelper.convertRawProviderToContentProvider(rawProvider.getName()); + + SecretEntity secret; + + try { + secret = secretRepository.findById(content.getSecret()); + } catch (RepositoryOperationFailureException e) { + throw new ContentApplicationRetrievalFailureException(e.getMessage()); + } + + CredentialsFieldsFull credentials = + repositoryConfigurationHelper.convertRawSecretsToContentCredentials( + provider, + secret.getSession(), + secret.getCredentials()); + + repositoryContentApplicationUnits.add( + RepositoryContentApplicationUnitDto.of( + content.getRoot(), provider, credentials)); + } + + return repositoryContentApplicationUnits; + } + /** * Removes temporate content from the temporate repository with the given hash. * diff --git a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java index ed572ce..453fc2a 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java @@ -1,18 +1,31 @@ package com.objectstorage.service.integration.backup; import com.objectstorage.converter.CronExpressionConverter; -import com.objectstorage.exception.BackupPeriodRetrievalFailureException; -import com.objectstorage.exception.CronExpressionException; +import com.objectstorage.converter.RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter; +import com.objectstorage.dto.FolderContentUnitDto; +import com.objectstorage.dto.RepositoryContentApplicationUnitDto; +import com.objectstorage.entity.common.PropertiesEntity; +import com.objectstorage.exception.*; +import com.objectstorage.model.ContentRetrievalProviderUnit; +import com.objectstorage.model.ValidationSecretsApplication; +import com.objectstorage.repository.facade.RepositoryFacade; import com.objectstorage.service.state.StateService; +import com.objectstorage.service.telemetry.TelemetryService; import com.objectstorage.service.vendor.VendorFacade; +import com.objectstorage.service.vendor.common.VendorConfigurationHelper; import com.objectstorage.service.workspace.facade.WorkspaceFacade; +import io.quarkus.runtime.Startup; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import com.objectstorage.service.config.ConfigService; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -20,8 +33,14 @@ /** * Provides backup configuration, which will create a local backup of uploaded files. */ +@Startup(value = 900) @ApplicationScoped public class BackupService { + private static final Logger logger = LogManager.getLogger(BackupService.class); + + @Inject + PropertiesEntity properties; + @Inject ConfigService configService; @@ -31,6 +50,12 @@ public class BackupService { @Inject VendorFacade vendorFacade; + @Inject + RepositoryFacade repositoryFacade; + + @Inject + TelemetryService telemetryService; + private final ScheduledExecutorService scheduledOperationExecutorService = Executors.newSingleThreadScheduledExecutor(); @@ -53,6 +78,86 @@ public void process() throws BackupPeriodRetrievalFailureException { scheduledOperationExecutorService.scheduleWithFixedDelay(() -> { StateService.getBackupProcessorGuard().lock(); + List repositoryContentApplicationUnits; + + try { + repositoryContentApplicationUnits = repositoryFacade.retrieveAllContentApplications(); + } catch (ContentApplicationRetrievalFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + + ValidationSecretsApplication validationSecretsApplication = + RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter + .convert(repositoryContentApplicationUnits); + + String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); + + for (RepositoryContentApplicationUnitDto repositoryContentApplicationUnit : + repositoryContentApplicationUnits) { + List contentRetrievalProviderUnits; + + try { + contentRetrievalProviderUnits = vendorFacade.listAllObjectsFromBucket( + repositoryContentApplicationUnit.getProvider(), + repositoryContentApplicationUnit.getCredentials().getExternal(), + VendorConfigurationHelper.createBucketName( + repositoryContentApplicationUnit.getRoot())); + } catch (SecretsConversionException | BucketObjectRetrievalFailureException | VendorOperationFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + + List folderContentUnits = new ArrayList<>(); + + for (ContentRetrievalProviderUnit contentRetrievalProviderUnit : contentRetrievalProviderUnits) { + byte[] content; + + try { + content = vendorFacade.retrieveObjectFromBucket( + repositoryContentApplicationUnit.getProvider(), + repositoryContentApplicationUnit.getCredentials().getExternal(), + VendorConfigurationHelper.createBucketName( + repositoryContentApplicationUnit.getRoot()), + contentRetrievalProviderUnit.getLocation() + ); + } catch ( + SecretsConversionException | + BucketObjectRetrievalFailureException | + VendorOperationFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + + folderContentUnits.add(FolderContentUnitDto.of( + contentRetrievalProviderUnit.getLocation(), content)); + } + + try { + workspaceFacade.addBackupFile( + workspaceUnitKey, + workspaceFacade.createFileUnitKey(properties.getWorkspaceContentBackupUnit()), + folderContentUnits); + } catch (FileCreationFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + + telemetryService.increaseCurrentBackupsAmount(); + } + StateService.getBackupProcessorGuard().unlock(); }, 0, period, TimeUnit.MILLISECONDS); } diff --git a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java index 57d4807..30251c4 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java @@ -1,12 +1,11 @@ package com.objectstorage.service.integration.temporatestorage; -import com.objectstorage.converter.ContentCompoundUnitToValidationSecretsApplicationConverter; +import com.objectstorage.converter.ContentCompoundUnitsToValidationSecretsApplicationConverter; import com.objectstorage.converter.CronExpressionConverter; import com.objectstorage.dto.ContentCompoundUnitDto; import com.objectstorage.dto.EarliestTemporateContentDto; import com.objectstorage.exception.*; import com.objectstorage.model.ValidationSecretsApplication; -import com.objectstorage.model.ValidationSecretsUnit; import com.objectstorage.repository.executor.RepositoryExecutor; import com.objectstorage.repository.facade.RepositoryFacade; import com.objectstorage.service.config.ConfigService; @@ -155,7 +154,7 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { } ValidationSecretsApplication validationSecretsApplication = - ContentCompoundUnitToValidationSecretsApplicationConverter.convert( + ContentCompoundUnitsToValidationSecretsApplicationConverter.convert( temporateContentDto.getContentCompoundUnits()); String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); @@ -192,7 +191,7 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { try { vendorFacade.uploadObjectToBucket( contentCompoundUnit.getProvider(), - contentCompoundUnit.getCredentialsFieldsFull().getExternal(), + contentCompoundUnit.getCredentials().getExternal(), VendorConfigurationHelper.createBucketName( contentCompoundUnit.getRepositoryContentUnitDto().getRoot()), temporateContentDto.getLocation(), diff --git a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java index 0afea18..5b4bedb 100644 --- a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java +++ b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java @@ -188,24 +188,6 @@ public void withdraw(ValidationSecretsApplication validationSecretsApplication) } for (ValidationSecretsUnit validationSecretsUnit : validationSecretsApplication.getSecrets()) { - RepositoryContentUnitDto repositoryContentLocationUnitDto; - - try { - repositoryContentLocationUnitDto = repositoryFacade.retrieveContentApplication(validationSecretsUnit); - } catch (ContentApplicationRetrievalFailureException e1) { - try { - repositoryExecutor.rollbackTransaction(); - } catch (TransactionRollbackFailureException e2) { - StateService.getTransactionProcessorGuard().unlock(); - - throw new ProcessorContentWithdrawalFailureException(e2.getMessage()); - } - - StateService.getTransactionProcessorGuard().unlock(); - - throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); - } - try { repositoryFacade.withdraw(validationSecretsUnit); } catch (RepositoryContentDestructionFailureException e1) { @@ -221,32 +203,6 @@ public void withdraw(ValidationSecretsApplication validationSecretsApplication) throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); } - - try { - if (vendorFacade.isBucketPresent( - validationSecretsUnit.getProvider(), - validationSecretsUnit.getCredentials().getExternal(), - VendorConfigurationHelper.createBucketName( - repositoryContentLocationUnitDto.getRoot()))) { - vendorFacade.removeBucket( - validationSecretsUnit.getProvider(), - validationSecretsUnit.getCredentials().getExternal(), - VendorConfigurationHelper.createBucketName( - repositoryContentLocationUnitDto.getRoot())); - } - } catch (SecretsConversionException | VendorOperationFailureException e1) { - try { - repositoryExecutor.rollbackTransaction(); - } catch (TransactionRollbackFailureException e2) { - StateService.getTransactionProcessorGuard().unlock(); - - throw new ProcessorContentWithdrawalFailureException(e2.getMessage()); - } - - StateService.getTransactionProcessorGuard().unlock(); - - throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); - } } try { diff --git a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java index be88cf3..b8c7f50 100644 --- a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java +++ b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java @@ -35,6 +35,8 @@ public class TelemetryService { private final ConcurrentLinkedQueue averageUploadFileSizeQueue = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue currentBackupsAmountQueue = new ConcurrentLinkedQueue<>(); + private final static ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(0, Thread.ofVirtual().factory()); @@ -66,6 +68,12 @@ private void configure() { averageUploadFileSizeQueue.poll().run(); } }, 0, properties.getDiagnosticsScrapeDelay(), TimeUnit.MILLISECONDS); + + scheduledExecutorService.scheduleWithFixedDelay(() -> { + if (!currentBackupsAmountQueue.isEmpty()) { + currentBackupsAmountQueue.poll().run(); + } + }, 0, properties.getDiagnosticsScrapeDelay(), TimeUnit.MILLISECONDS); } /** @@ -113,4 +121,15 @@ public void setAverageUploadFileSizeQueue(Double value) { () -> telemetryBinding.getAverageUploadFileSize().set(value)); } } + + /** + * Increases performed cloud service backup operations. + */ + public void increaseCurrentBackupsAmount() { + if (configService.getConfig().getDiagnostics().getEnabled()) { + currentBackupsAmountQueue.add( + () -> telemetryBinding.getCurrentBackupsAmount().set( + telemetryBinding.getCurrentBackupsAmount().get() + 1)); + } + } } diff --git a/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java b/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java index 6c6b8f4..7b51ec1 100644 --- a/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java +++ b/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java @@ -28,6 +28,8 @@ public class TelemetryBinding implements MeterBinder { private final AtomicDouble averageUploadFileSize = new AtomicDouble(); + private final AtomicInteger currentBackupsAmount = new AtomicInteger(); + /** * @see MeterBinder */ @@ -38,7 +40,7 @@ public void bindTo(@NotNull MeterRegistry meterRegistry) { .register(meterRegistry); Gauge.builder("general.current_cloud_service_uploads_amount", currentCloudServiceUploadsAmount, AtomicInteger::get) - .description("Represents amount of uploads to different cloud services") + .description("Represents amount of uploads to different cloud services in the current session") .register(meterRegistry); Gauge.builder("general.cloud_service_upload_retries", cloudServiceUploadRetries, AtomicInteger::get) @@ -52,5 +54,9 @@ public void bindTo(@NotNull MeterRegistry meterRegistry) { Gauge.builder("general.average_upload_file_size", averageUploadFileSize, AtomicDouble::get) .description("Represents average upload file size in ObjectStorage Temporate Storage") .register(meterRegistry); + + Gauge.builder("general.current_backups_amount", currentBackupsAmount, AtomicInteger::get) + .description("Represents amount of performed cloud service backup operation in the current session") + .register(meterRegistry); } } diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java b/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java index a9da9a8..fa79435 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java @@ -1,8 +1,10 @@ package com.objectstorage.service.workspace; +import com.objectstorage.dto.FolderContentUnitDto; import com.objectstorage.entity.common.PropertiesEntity; import com.objectstorage.exception.*; import com.objectstorage.exception.FileNotFoundException; +import com.objectstorage.service.workspace.common.WorkspaceConfigurationHelper; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import jakarta.xml.bind.DatatypeConverter; @@ -19,6 +21,7 @@ import java.util.List; import java.util.Objects; import java.util.stream.Stream; +import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @@ -316,6 +319,42 @@ public byte[] compressFile(InputStream inputStream) throws return result.toByteArray(); } + /** + * Compresses given folder entities of the given type. This will act as a non-compressed folder, which has + * previously compressed with zip files. + * + * @param folderContentUnits given folder input entities. + * @param type given folder type. + * @return compressed folder input. + * @throws InputCompressionFailureException if input compression fails. + */ + public byte[] compressFolder(List folderContentUnits, String type) throws + InputCompressionFailureException { + ByteArrayOutputStream result = new ByteArrayOutputStream(); + + try (ZipOutputStream writer = new ZipOutputStream(result)) { + writer.setMethod(ZipOutputStream.DEFLATED); + writer.setLevel(Deflater.NO_COMPRESSION); + + writer.putNextEntry(new ZipEntry(WorkspaceConfigurationHelper.getZipFolderDefinition(type))); + + for (FolderContentUnitDto folderContentUnit : folderContentUnits) { + writer.putNextEntry(new ZipEntry(folderContentUnit.getLocation())); + + writer.write(folderContentUnit.getContent()); + } + + writer.flush(); + + writer.finish(); + + } catch (IOException e) { + throw new InputCompressionFailureException(e.getMessage()); + } + + return result.toByteArray(); + } + /** * Adds new file of the given type to the workspace with the given workspace unit key as the compressed input stream. * diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java b/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java new file mode 100644 index 0000000..41cc530 --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java @@ -0,0 +1,17 @@ +package com.objectstorage.service.workspace.common; + +/** + * Contains helpful tools used for workspace configuration. + */ +public class WorkspaceConfigurationHelper { + + /** + * Creates folder definition with the help of the given folder name for ZIP achieve. + * + * @param name given folder name value. + * @return wrapped token. + */ + public static String getZipFolderDefinition(String name) { + return String.format("%s/", name); + } +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java index 35e010a..48a8759 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java @@ -1,5 +1,6 @@ package com.objectstorage.service.workspace.facade; +import com.objectstorage.dto.FolderContentUnitDto; import com.objectstorage.entity.common.PropertiesEntity; import com.objectstorage.exception.*; import com.objectstorage.model.ValidationSecretsApplication; @@ -10,6 +11,7 @@ import java.io.*; import java.time.Instant; +import java.util.List; import java.util.stream.Collectors; /** @@ -55,7 +57,12 @@ public String createWorkspaceUnitKey(ValidationSecretsApplication validationSecr * @return created file unit key. */ public String createFileUnitKey(String name) { - return workspaceService.createUnitKey(name, Instant.now().toString()); + Instant timestamp = Instant.now(); + + String fileUnit = + workspaceService.createUnitKey(name, Instant.now().toString()); + + return String.format("%s-%s-%d", name, fileUnit, timestamp.toEpochMilli()); } /** @@ -75,13 +82,26 @@ public void addObjectFile(String workspaceUnitKey, String name, InputStream inpu * Adds new backup file to the workspace with the given workspace unit key as the compressed input stream. * * @param workspaceUnitKey given user workspace unit key. - * @param name given content name. - * @param inputStream given input. + * @param name given file name. + * @param folderContentUnits given folder content units. * @throws FileCreationFailureException if file creation operation failed. */ - public void addBackupFile(String workspaceUnitKey, String name, InputStream inputStream) + public void addBackupFile(String workspaceUnitKey, String name, List folderContentUnits) throws FileCreationFailureException { - workspaceService.addContentFile(workspaceUnitKey, properties.getWorkspaceContentBackupDirectory(), name, inputStream); + byte[] content; + + try { + content = + workspaceService.compressFolder(folderContentUnits, properties.getWorkspaceContentBackupDirectory()); + } catch (InputCompressionFailureException e) { + throw new FileCreationFailureException(e.getMessage()); + } + + workspaceService.addContentFile( + workspaceUnitKey, + properties.getWorkspaceContentBackupDirectory(), + name, + new ByteArrayInputStream(content)); Integer amount; @@ -166,16 +186,6 @@ public void removeObjectFile(String workspaceUnitKey, String name) throws FileRe workspaceService.removeContentFile(workspaceUnitKey, properties.getWorkspaceContentObjectDirectory(), name); } - /** - * Removes backup file with the given name from the workspace with the help of the given workspace unit key. - * - * @param workspaceUnitKey given user workspace unit key. - * @throws FileRemovalFailureException if file removal operation failed. - */ - public void removeBackupFile(String workspaceUnitKey, String name) throws FileRemovalFailureException { - workspaceService.removeContentFile(workspaceUnitKey, properties.getWorkspaceContentBackupDirectory(), name); - } - /** * Removes all the files from the workspace with the help of the given workspace unit key. * diff --git a/api-server/src/main/openapi/openapi.yml b/api-server/src/main/openapi/openapi.yml index b89e1f0..15a7eb9 100644 --- a/api-server/src/main/openapi/openapi.yml +++ b/api-server/src/main/openapi/openapi.yml @@ -264,6 +264,7 @@ components: required: - pending - uploaded + - backups properties: pending: type: array diff --git a/api-server/src/main/resources/application.properties b/api-server/src/main/resources/application.properties index f7dd05d..d9213f4 100644 --- a/api-server/src/main/resources/application.properties +++ b/api-server/src/main/resources/application.properties @@ -74,6 +74,9 @@ workspace.content.object.directory=object # Describes location of backup content directory. workspace.content.backup.directory=backup +# Describes location of backup content unit. +workspace.content.backup.unit=backup + # Describes name of the file used for compression operation. workspace.compression.file.name=objectstorage-file From a54a5deff1e4a778210470706324c03a4a5d3ea0 Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi Date: Fri, 29 Nov 2024 15:54:29 +0100 Subject: [PATCH 9/9] fix: fixed bugs --- ...itsLocationsRetrievalFailureException.java | 21 --- .../FileUnitsRetrievalFailureException.java | 21 +++ .../service/processor/ProcessorService.java | 12 +- .../service/workspace/WorkspaceService.java | 56 ++++++-- .../common/WorkspaceConfigurationHelper.java | 10 ++ .../workspace/facade/WorkspaceFacade.java | 32 ++++- api-server/src/main/openapi/openapi.yml | 10 ++ config/grafana/dashboards/diagnostics.tmpl | 124 ++++++++++++++++-- 8 files changed, 237 insertions(+), 49 deletions(-) delete mode 100644 api-server/src/main/java/com/objectstorage/exception/FileUnitsLocationsRetrievalFailureException.java create mode 100644 api-server/src/main/java/com/objectstorage/exception/FileUnitsRetrievalFailureException.java diff --git a/api-server/src/main/java/com/objectstorage/exception/FileUnitsLocationsRetrievalFailureException.java b/api-server/src/main/java/com/objectstorage/exception/FileUnitsLocationsRetrievalFailureException.java deleted file mode 100644 index 310dc1c..0000000 --- a/api-server/src/main/java/com/objectstorage/exception/FileUnitsLocationsRetrievalFailureException.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.objectstorage.exception; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Formatter; - -/** - * Represents exception used when file units locations retrieval operation fails. - */ -public class FileUnitsLocationsRetrievalFailureException extends IOException { - public FileUnitsLocationsRetrievalFailureException() { - this(""); - } - - public FileUnitsLocationsRetrievalFailureException(Object... message) { - super( - new Formatter() - .format("File units locations retrieval operation failed: %s", Arrays.stream(message).toArray()) - .toString()); - } -} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/exception/FileUnitsRetrievalFailureException.java b/api-server/src/main/java/com/objectstorage/exception/FileUnitsRetrievalFailureException.java new file mode 100644 index 0000000..ba8de4e --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/exception/FileUnitsRetrievalFailureException.java @@ -0,0 +1,21 @@ +package com.objectstorage.exception; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Formatter; + +/** + * Represents exception used when file units retrieval operation fails. + */ +public class FileUnitsRetrievalFailureException extends IOException { + public FileUnitsRetrievalFailureException() { + this(""); + } + + public FileUnitsRetrievalFailureException(Object... message) { + super( + new Formatter() + .format("File units retrieval operation failed: %s", Arrays.stream(message).toArray()) + .toString()); + } +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java index 5b4bedb..00d0f39 100644 --- a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java +++ b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java @@ -55,6 +55,8 @@ public ContentRetrievalResult retrieveContent(ValidationSecretsApplication valid throws ProcessorContentRetrievalFailureException { List compounds = new ArrayList<>(); + String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); + for (ValidationSecretsUnit validationSecretsUnit : validationSecretsApplication.getSecrets()) { RepositoryContentUnitDto repositoryContentLocationUnitDto; @@ -84,11 +86,19 @@ public ContentRetrievalResult retrieveContent(ValidationSecretsApplication valid VendorOperationFailureException ignored) { } + List backups; + + try { + backups = workspaceFacade.getBackupUnits(workspaceUnitKey); + } catch (FileUnitsRetrievalFailureException e) { + throw new ProcessorContentRetrievalFailureException(e.getMessage()); + } + compounds.add( ContentRetrievalCompound.of( repositoryContentLocationUnitDto.getRoot(), validationSecretsUnit.getProvider().toString(), - List.of(ContentRetrievalUnits.of(pending, uploaded)))); + List.of(ContentRetrievalUnits.of(pending, uploaded, backups)))); } return ContentRetrievalResult.of(compounds); diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java b/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java index fa79435..b59b925 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java @@ -23,7 +23,6 @@ import java.util.stream.Stream; import java.util.zip.Deflater; import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; import lombok.SneakyThrows; @@ -95,7 +94,7 @@ public void createContentDirectory(String workspaceUnitDirectory, String type) t * @param key given workspace unit directory. * @return result of the check. */ - public Boolean isUnitDirectoryExist(String key) { + private Boolean isUnitDirectoryExist(String key) { return Files.exists(Paths.get(properties.getWorkspaceDirectory(), key)); } @@ -106,7 +105,7 @@ public Boolean isUnitDirectoryExist(String key) { * @param type given content directory type. * @return result of the check. */ - public Boolean isContentDirectoryExist(String workspaceUnitDirectory, String type) { + private Boolean isContentDirectoryExist(String workspaceUnitDirectory, String type) { return Files.exists(Paths.get(workspaceUnitDirectory, type)); } @@ -339,7 +338,11 @@ public byte[] compressFolder(List folderContentUnits, Stri writer.putNextEntry(new ZipEntry(WorkspaceConfigurationHelper.getZipFolderDefinition(type))); for (FolderContentUnitDto folderContentUnit : folderContentUnits) { - writer.putNextEntry(new ZipEntry(folderContentUnit.getLocation())); + writer.putNextEntry(new ZipEntry( + Path.of( + properties.getWorkspaceContentBackupDirectory(), + WorkspaceConfigurationHelper.getZipFile( + folderContentUnit.getLocation())).toString())); writer.write(folderContentUnit.getContent()); } @@ -361,10 +364,10 @@ public byte[] compressFolder(List folderContentUnits, Stri * @param workspaceUnitKey given user workspace unit key. * @param type given content type. * @param name given content name. - * @param inputStream given input. + * @param content given content. * @throws FileCreationFailureException if file creation operation failed. */ - public void addContentFile(String workspaceUnitKey, String type, String name, InputStream inputStream) + public void addContentFile(String workspaceUnitKey, String type, String name, byte[] content) throws FileCreationFailureException { if (!isUnitDirectoryExist(workspaceUnitKey)) { try { @@ -394,14 +397,6 @@ public void addContentFile(String workspaceUnitKey, String type, String name, In throw new FileCreationFailureException(); } - byte[] content; - - try { - content = compressFile(inputStream); - } catch (InputCompressionFailureException e) { - throw new FileCreationFailureException(e.getMessage()); - } - try { createFile(workspaceUnitDirectory, type, name, content); } catch (FileWriteFailureException e) { @@ -435,6 +430,39 @@ public Boolean isContentFilePresent(String workspaceUnitKey, String type, String return false; } + /** + * Retrieves content units in the workspace with the given workspace unit key and of the given type. + * + * @param workspaceUnitKey given user workspace unit key. + * @param type given file type. + * @return retrieves additional content units. + * @throws FileUnitsRetrievalFailureException if content units retrieval failed. + */ + public List getContentUnits(String workspaceUnitKey, String type) throws + FileUnitsRetrievalFailureException { + List result = new ArrayList<>(); + + if (isUnitDirectoryExist(workspaceUnitKey)) { + String workspaceUnitDirectory; + + try { + workspaceUnitDirectory = getUnitDirectory(workspaceUnitKey); + } catch (WorkspaceUnitDirectoryNotFoundException e) { + throw new FileUnitsRetrievalFailureException(e.getMessage()); + } + + if (isContentDirectoryExist(workspaceUnitDirectory, type)) { + try { + result = getFilesLocations(workspaceUnitDirectory, type); + } catch (FilesLocationsRetrievalFailureException e) { + throw new FileUnitsRetrievalFailureException(e.getMessage()); + } + } + } + + return result; + } + /** * Retrieves file from the workspace with the given workspace unit key as compressed byte array. * diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java b/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java index 41cc530..f95e0de 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java @@ -14,4 +14,14 @@ public class WorkspaceConfigurationHelper { public static String getZipFolderDefinition(String name) { return String.format("%s/", name); } + + /** + * Creates zip file definition with the help of the given file name. + * + * @param name given zip file name. + * @return wrapped zip file. + */ + public static String getZipFile(String name) { + return String.format("%s.zip", name); + } } \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java index 48a8759..6d90ec5 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java @@ -3,6 +3,7 @@ import com.objectstorage.dto.FolderContentUnitDto; import com.objectstorage.entity.common.PropertiesEntity; import com.objectstorage.exception.*; +import com.objectstorage.model.ContentRetrievalBackupUnit; import com.objectstorage.model.ValidationSecretsApplication; import com.objectstorage.service.config.ConfigService; import com.objectstorage.service.workspace.WorkspaceService; @@ -75,7 +76,19 @@ public String createFileUnitKey(String name) { */ public void addObjectFile(String workspaceUnitKey, String name, InputStream inputStream) throws FileCreationFailureException { - workspaceService.addContentFile(workspaceUnitKey, properties.getWorkspaceContentObjectDirectory(), name, inputStream); + byte[] content; + + try { + content = workspaceService.compressFile(inputStream); + } catch (InputCompressionFailureException e) { + throw new FileCreationFailureException(e.getMessage()); + } + + workspaceService.addContentFile( + workspaceUnitKey, + properties.getWorkspaceContentObjectDirectory(), + name, + content); } /** @@ -101,7 +114,7 @@ public void addBackupFile(String workspaceUnitKey, String name, List getBackupUnits(String workspaceUnitKey) throws FileUnitsRetrievalFailureException { + return workspaceService + .getContentUnits(workspaceUnitKey, properties.getWorkspaceContentBackupDirectory()) + .stream() + .map(ContentRetrievalBackupUnit::of) + .toList(); + } + /** * Retrieves object file with the given name and of the given type from the workspace with the given workspace * unit key as compressed byte array. diff --git a/api-server/src/main/openapi/openapi.yml b/api-server/src/main/openapi/openapi.yml index 15a7eb9..f6092e3 100644 --- a/api-server/src/main/openapi/openapi.yml +++ b/api-server/src/main/openapi/openapi.yml @@ -274,6 +274,10 @@ components: type: array items: $ref: "#/components/schemas/ContentRetrievalProviderUnit" + backups: + type: array + items: + $ref: "#/components/schemas/ContentRetrievalBackupUnit" ContentRetrievalProviderUnit: required: - location @@ -284,6 +288,12 @@ components: created_at: type: integer format: int64 + ContentRetrievalBackupUnit: + required: + - location + properties: + location: + type: string ContentApplication: required: - root diff --git a/config/grafana/dashboards/diagnostics.tmpl b/config/grafana/dashboards/diagnostics.tmpl index 44d81a0..a9e7da5 100644 --- a/config/grafana/dashboards/diagnostics.tmpl +++ b/config/grafana/dashboards/diagnostics.tmpl @@ -440,6 +440,108 @@ "title": "ObjectStorage API Server average upload file size", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "P21B111CBFE6E8FCA" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 38, + "interval": "0.1", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P21B111CBFE6E8FCA" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "general_current_backups_amount", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "ObjectStorage API Server current backups amount", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -466,7 +568,7 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, + "x": 12, "y": 17 }, "id": 32, @@ -538,8 +640,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 17 + "x": 0, + "y": 25 }, "id": 35, "options": { @@ -587,7 +689,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 33 }, "id": 28, "panels": [], @@ -636,7 +738,7 @@ "h": 7, "w": 3, "x": 0, - "y": 26 + "y": 34 }, "id": 15, "maxDataPoints": 100, @@ -721,7 +823,7 @@ "h": 6, "w": 6, "x": 3, - "y": 26 + "y": 34 }, "id": 6, "maxDataPoints": 100, @@ -807,7 +909,7 @@ "h": 6, "w": 6, "x": 9, - "y": 26 + "y": 34 }, "id": 4, "maxDataPoints": 100, @@ -845,7 +947,7 @@ "type": "gauge" } ], - "refresh": "5s", + "refresh": "", "schemaVersion": 39, "tags": [ "docker", @@ -1021,8 +1123,8 @@ ] }, "time": { - "from": "now-5m", - "to": "now" + "from": "2024-11-29T14:23:10.911Z", + "to": "2024-11-29T14:25:03.115Z" }, "timepicker": { "refresh_intervals": [ @@ -1052,6 +1154,6 @@ "timezone": "browser", "title": "ObjectStorage Diagnostics", "uid": "64nrElFmk", - "version": 7, + "version": 10, "weekStart": "" } \ No newline at end of file