From 4e669a53d5f402699c40bcb13a7c3d4694763d35 Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 14 Jul 2021 12:48:58 +0000 Subject: [PATCH 01/62] Apply patch for double prefix --- Model/Adapter/StorageObjectManagement.php | 2 +- composer.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 0decdb4..609cada 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -313,7 +313,7 @@ public function uploadObject($handle, array $options = []): ?StorageObject if (isset($options['name'])) { $options['name'] = implode(DIRECTORY_SEPARATOR, [ $prefix, - ltrim($options['name'], DIRECTORY_SEPARATOR), + ltrim($options['name'], DIRECTORY_SEPARATOR . $prefix), ]); } else { /** @var StreamInterface $stream */ diff --git a/composer.json b/composer.json index 291c604..2488825 100644 --- a/composer.json +++ b/composer.json @@ -1,5 +1,5 @@ { - "name": "auroraextensions/googlecloudstorage", + "name": "outeredge/googlecloudstorage", "description": "Google Cloud Storage integration for Magento.", "type": "magento2-module", "license": "MIT", From a39eba284279bc0173699af0f2d1c5085067c084 Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 14 Jul 2021 16:09:16 +0100 Subject: [PATCH 02/62] Fix missing DIRECTORY_SEPARATOR --- Plugin/Framework/File/Uploader/ObjectUploader.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Plugin/Framework/File/Uploader/ObjectUploader.php b/Plugin/Framework/File/Uploader/ObjectUploader.php index 27ca1d7..9810616 100644 --- a/Plugin/Framework/File/Uploader/ObjectUploader.php +++ b/Plugin/Framework/File/Uploader/ObjectUploader.php @@ -95,7 +95,7 @@ public function afterSave( $baseName = $result['file'] ?? ''; /** @var string $realPath */ - $realPath = $basePath . $baseName; + $realPath = $basePath . DIRECTORY_SEPARATOR . $baseName; if (!empty($realPath)) { $this->upload($realPath); From e0e0fd2db4a9b6280da96b66d9fdc856a6ec47ef Mon Sep 17 00:00:00 2001 From: David Windell Date: Tue, 7 Dec 2021 19:05:53 +0000 Subject: [PATCH 03/62] Add get.php support --- Model/File/Storage/Bucket.php | 3 ++ Model/File/Storage/Synchronization.php | 52 ++++++++++++++++++++++++++ etc/di.xml | 3 ++ 3 files changed, 58 insertions(+) create mode 100644 Model/File/Storage/Synchronization.php diff --git a/Model/File/Storage/Bucket.php b/Model/File/Storage/Bucket.php index bc6197e..56a8994 100644 --- a/Model/File/Storage/Bucket.php +++ b/Model/File/Storage/Bucket.php @@ -157,6 +157,9 @@ public function loadByFilename(string $filename) /** @var string $relativePath */ $relativePath = $this->storageHelper->getMediaRelativePath($filename); + // @todo look up media path like https://github.com/magento/magento2/blob/2.4-develop/app/code/Magento/MediaStorage/App/Media.php#L187 + $relativePath = str_replace('media/', '', $relativePath); + if ($this->getStorage()->objectExists($relativePath)) { $this->setData('id', $filename); $this->setData('filename', $filename); diff --git a/Model/File/Storage/Synchronization.php b/Model/File/Storage/Synchronization.php new file mode 100644 index 0000000..c840767 --- /dev/null +++ b/Model/File/Storage/Synchronization.php @@ -0,0 +1,52 @@ +mediaDirectory = $directory; + $this->bucket = $bucket; + } + + /** + * Synchronize file from GCS to local filesystem + * + * @param string $relativeFileName + * @return void + */ + public function synchronize($relativeFileName) + { + $storage = $this->bucket->loadByFilename($relativeFileName); + + if ($storage->getId()) { + $file = $this->mediaDirectory->openFile($relativeFileName, 'w'); + try { + $file->lock(); + $file->write($storage->getContent()); + $file->unlock(); + $file->close(); + } catch (FileSystemException $e) { + $file->close(); + } + } + } +} diff --git a/etc/di.xml b/etc/di.xml index 8089cd6..0d306d9 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -20,6 +20,9 @@ + + From 8e7d4298b909e1ca0deaf8be5cc5292052d52584 Mon Sep 17 00:00:00 2001 From: David Windell Date: Tue, 7 Dec 2021 19:19:31 +0000 Subject: [PATCH 04/62] Improve factory definition to match core --- Model/File/Storage/Synchronization.php | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Model/File/Storage/Synchronization.php b/Model/File/Storage/Synchronization.php index c840767..c9c0bf5 100644 --- a/Model/File/Storage/Synchronization.php +++ b/Model/File/Storage/Synchronization.php @@ -10,7 +10,7 @@ class Synchronization /** * @var BucketFactory */ - protected $bucket; + protected $storageFactory; /** * File stream handler @@ -20,11 +20,11 @@ class Synchronization protected $mediaDirectory; public function __construct( - DirectoryWrite $directory, - BucketFactory $bucket + BucketFactory $storageFactory, + DirectoryWrite $directory ) { + $this->storageFactory = $storageFactory; $this->mediaDirectory = $directory; - $this->bucket = $bucket; } /** @@ -35,9 +35,14 @@ public function __construct( */ public function synchronize($relativeFileName) { - $storage = $this->bucket->loadByFilename($relativeFileName); - + /** @var $storage Bucket */ + $storage = $this->storageFactory->create(); + try { + $storage->loadByFilename($relativeFileName); + } catch (\Exception $e) { + } if ($storage->getId()) { + /** @var WriteInterface $file */ $file = $this->mediaDirectory->openFile($relativeFileName, 'w'); try { $file->lock(); From afdb58c8c9db8b2cd0271a370f63c105cf0a9eca Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 8 Dec 2021 19:30:08 +0000 Subject: [PATCH 05/62] Add product image sync plugin --- Plugin/Catalog/Product/Image.php | 66 ++++++++++++++++++++++++++++++++ etc/di.xml | 5 +++ 2 files changed, 71 insertions(+) create mode 100644 Plugin/Catalog/Product/Image.php diff --git a/Plugin/Catalog/Product/Image.php b/Plugin/Catalog/Product/Image.php new file mode 100644 index 0000000..c74f189 --- /dev/null +++ b/Plugin/Catalog/Product/Image.php @@ -0,0 +1,66 @@ +mediaDirectory = $filesystem->getDirectoryWrite(DirectoryList::MEDIA); + $this->mediaConfig = $mediaConfig; + $this->storageFactory = $storageFactory; + } + + public function beforeSetBaseFile(ProductImage $image, $file) + { + $relativeFileName = $this->mediaConfig->getBaseMediaPath() . $file; + + if ($this->mediaDirectory->isFile($relativeFileName)) { + return; + } + + /** @var $storage Storage\Bucket */ + $storage = $this->storageFactory->create(); + try { + $storage->loadByFilename($relativeFileName); + } catch (\Exception $e) { + } + if ($storage->getId()) { + /** @var WriteInterface $file */ + $file = $this->mediaDirectory->openFile($relativeFileName, 'w'); + try { + $file->lock(); + $file->write($storage->getContent()); + $file->unlock(); + $file->close(); + } catch (FileSystemException $e) { + $file->close(); + } + } + } +} diff --git a/etc/di.xml b/etc/di.xml index 0d306d9..e8edb9c 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -110,4 +110,9 @@ type="AuroraExtensions\GoogleCloudStorage\Plugin\MediaStorage\Config\SynchronizeStorageParams" sortOrder="10"/> + + + From a7ac6ea253be9fa09e53b86feba790f4ad98bab9 Mon Sep 17 00:00:00 2001 From: David Windell Date: Thu, 9 Dec 2021 18:30:34 +0000 Subject: [PATCH 06/62] Fix ltrim issue --- Model/Adapter/StorageObjectManagement.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index d039d48..f5b0f01 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -313,7 +313,7 @@ public function uploadObject($handle, array $options = []): ?StorageObject if (isset($options['name'])) { $options['name'] = implode(DIRECTORY_SEPARATOR, [ $prefix, - ltrim($options['name'], DIRECTORY_SEPARATOR . $prefix), + str_replace(DIRECTORY_SEPARATOR . $prefix . DIRECTORY_SEPARATOR, '', $options['name']), ]); } else { /** @var StreamInterface $stream */ From 82f5e57b0d9449929ede83b91c293ad4741ee08a Mon Sep 17 00:00:00 2001 From: David Windell Date: Fri, 10 Dec 2021 17:31:01 +0000 Subject: [PATCH 07/62] Add enabled/disabled check --- Api/StorageObjectManagementInterface.php | 5 +++++ Model/Adapter/StorageObjectManagement.php | 20 ++++++++++++++++++- .../Product/Gallery/ObjectUploader.php | 4 ++++ Plugin/Catalog/Product/Image.php | 9 +++++++-- .../File/Uploader/ObjectUploader.php | 6 +++++- .../Framework/Image/Adapter/ObjectAdapter.php | 4 ++++ .../File/Storage/FileProcessor.php | 2 +- 7 files changed, 45 insertions(+), 5 deletions(-) diff --git a/Api/StorageObjectManagementInterface.php b/Api/StorageObjectManagementInterface.php index 4b62bd9..4bda908 100644 --- a/Api/StorageObjectManagementInterface.php +++ b/Api/StorageObjectManagementInterface.php @@ -67,4 +67,9 @@ public function deleteObject(string $path): bool; * @return \AuroraExtensions\GoogleCloudStorage\Api\StorageObjectManagementInterface */ public function deleteAllObjects(array $options): StorageObjectManagementInterface; + + /** + * @return bool + */ + public function isEnabled(): bool; } diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index f5b0f01..3b923e4 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -23,7 +23,8 @@ Api\StorageObjectPathResolverInterface, Component\ModuleConfigTrait, Exception\InvalidGoogleCloudStorageSetupException, - Model\System\ModuleConfig + Model\System\ModuleConfig, + Model\File\Storage }; use AuroraExtensions\ModuleComponents\{ Api\LocalizedScopeDeploymentConfigInterface, @@ -92,6 +93,9 @@ class StorageObjectManagement implements StorageObjectManagementInterface, Stora /** @var bool $useModuleConfig */ private $useModuleConfig; + /** @var bool $enabled */ + private $enabled = false; + /** * @param LocalizedScopeDeploymentConfigInterfaceFactory $deploymentConfigFactory * @param ExceptionFactory $exceptionFactory @@ -105,12 +109,14 @@ class StorageObjectManagement implements StorageObjectManagementInterface, Stora public function __construct( LocalizedScopeDeploymentConfigInterfaceFactory $deploymentConfigFactory, ExceptionFactory $exceptionFactory, + Storage $fileStorage, FileDriver $fileDriver, Filesystem $filesystem, ModuleConfig $moduleConfig, StreamInterfaceFactory $streamFactory, bool $useModuleConfig = false ) { + $this->enabled = $fileStorage->checkBucketUsage(); $this->deploymentConfig = $deploymentConfigFactory->create(['scope' => 'googlecloud']); $this->exceptionFactory = $exceptionFactory; $this->fileDriver = $fileDriver; @@ -127,6 +133,10 @@ public function __construct( */ private function initialize(): void { + if (!$this->isEnabled()) { + return; + } + /** @var string|null $projectName */ $projectName = $this->useModuleConfig ? $this->getConfig()->getGoogleCloudProject() @@ -451,4 +461,12 @@ public function getObjectAclPolicy(): string return !empty($aclPolicy) ? $aclPolicy : ModuleConfig::DEFAULT_ACL_POLICY; } + + /** + * {@inheritdoc} + */ + public function isEnabled() : bool + { + return $this->enabled; + } } diff --git a/Plugin/Catalog/Product/Gallery/ObjectUploader.php b/Plugin/Catalog/Product/Gallery/ObjectUploader.php index 514f4d0..e887b4e 100644 --- a/Plugin/Catalog/Product/Gallery/ObjectUploader.php +++ b/Plugin/Catalog/Product/Gallery/ObjectUploader.php @@ -97,6 +97,10 @@ public function afterExecute( ExtensionInterface $subject, ProductInterface $result ) { + if (!$this->storageAdapter->isEnabled()) { + return $result; + } + /** @var string $attrCode */ $attrCode = $subject->getAttribute() ->getAttributeCode(); diff --git a/Plugin/Catalog/Product/Image.php b/Plugin/Catalog/Product/Image.php index c74f189..91a80df 100644 --- a/Plugin/Catalog/Product/Image.php +++ b/Plugin/Catalog/Product/Image.php @@ -38,14 +38,19 @@ public function __construct( public function beforeSetBaseFile(ProductImage $image, $file) { + /** @var $storage Storage\Bucket */ + $storage = $this->storageFactory->create(); + + if (!$storage->getStorage()->isEnabled()) { + return; + } + $relativeFileName = $this->mediaConfig->getBaseMediaPath() . $file; if ($this->mediaDirectory->isFile($relativeFileName)) { return; } - /** @var $storage Storage\Bucket */ - $storage = $this->storageFactory->create(); try { $storage->loadByFilename($relativeFileName); } catch (\Exception $e) { diff --git a/Plugin/Framework/File/Uploader/ObjectUploader.php b/Plugin/Framework/File/Uploader/ObjectUploader.php index 6b4cd96..1714074 100644 --- a/Plugin/Framework/File/Uploader/ObjectUploader.php +++ b/Plugin/Framework/File/Uploader/ObjectUploader.php @@ -95,6 +95,10 @@ public function afterSave( $destinationFolder, $newFileName = null ) { + if (!$this->storageAdapter->isEnabled()) { + return $result; + } + if (!empty($result)) { /** @var string $basePath */ $basePath = (string)($result['path'] ?? ''); @@ -141,4 +145,4 @@ private function upload(string $path): void $this->logger->critical($e->getMessage()); } } -} \ No newline at end of file +} diff --git a/Plugin/Framework/Image/Adapter/ObjectAdapter.php b/Plugin/Framework/Image/Adapter/ObjectAdapter.php index 23f5b33..a9d8285 100644 --- a/Plugin/Framework/Image/Adapter/ObjectAdapter.php +++ b/Plugin/Framework/Image/Adapter/ObjectAdapter.php @@ -90,6 +90,10 @@ public function afterSave( $destination = null, $newName = null ) { + if (!$this->storageAdapter->isEnabled()) { + return; + } + if (!empty($destination)) { /** @var string $filePath */ $filePath = $this->storageHelper->getMediaRelativePath($destination); diff --git a/Plugin/MediaStorage/File/Storage/FileProcessor.php b/Plugin/MediaStorage/File/Storage/FileProcessor.php index 2535465..5396f91 100644 --- a/Plugin/MediaStorage/File/Storage/FileProcessor.php +++ b/Plugin/MediaStorage/File/Storage/FileProcessor.php @@ -73,7 +73,7 @@ public function afterSaveFile( $file, $overwrite = true ) { - if (!$result) { + if (!$result || !$this->storageAdapter->isEnabled()) { return $result; } From 5d1f661861ec5cf83a522411dd098c3e2e12c08a Mon Sep 17 00:00:00 2001 From: David Windell Date: Fri, 10 Dec 2021 17:48:16 +0000 Subject: [PATCH 08/62] Skip uploading /tmp files --- Model/Adapter/StorageObjectManagement.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 3b923e4..22463ed 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -350,6 +350,10 @@ public function uploadObject($handle, array $options = []): ?StorageObject } } + if (stristr($options['name'], DIRECTORY_SEPARATOR . 'tmp' . DIRECTORY_SEPARATOR)) { + return null; + } + return $this->bucket->upload($handle, $options); } From b1355835815c54089e9a7b47954e0278c3881b17 Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 15 Dec 2021 14:14:09 +0000 Subject: [PATCH 09/62] Skip sync if storage is not enabled --- Model/File/Storage/Synchronization.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Model/File/Storage/Synchronization.php b/Model/File/Storage/Synchronization.php index c9c0bf5..1a63b23 100644 --- a/Model/File/Storage/Synchronization.php +++ b/Model/File/Storage/Synchronization.php @@ -37,6 +37,11 @@ public function synchronize($relativeFileName) { /** @var $storage Bucket */ $storage = $this->storageFactory->create(); + + if (!$storage->getStorage()->isEnabled()) { + return; + } + try { $storage->loadByFilename($relativeFileName); } catch (\Exception $e) { From 2f07b50cbbdeaa5f917a4910b80bbea2338fb898 Mon Sep 17 00:00:00 2001 From: David Windell Date: Mon, 24 Jan 2022 18:06:25 +0000 Subject: [PATCH 10/62] Add fallback_url for missing images --- Model/Adapter/StorageObjectManagement.php | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 22463ed..c951e57 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -272,7 +272,25 @@ public function getObject(string $path): ?StorageObject ]); } - return $this->bucket->object($path); + $object = $this->bucket->object($path); + + if (!$object && $fallback = $this->deploymentConfig->get('storage/fallback_url')) { + /* Attempt to load the image from fallback URL and upload to GCS */ + $ch = curl_init($fallback . $path); + curl_setopt($ch, CURLOPT_TIMEOUT, 5); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $content = curl_exec($ch); + curl_close($ch); + + if ($content) { + $object = $this->uploadObject($content, [ + 'name' => $path, + 'predefinedAcl' => $this->getObjectAclPolicy() + ]); + } + } + + return $object; } /** From e9923333f628ee0cffac84ca37b73051d1828845 Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 26 Jan 2022 13:49:25 +0000 Subject: [PATCH 11/62] Update StorageObjectManagement.php --- Model/Adapter/StorageObjectManagement.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index c951e57..0b9a6b2 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -266,13 +266,13 @@ public function getClient(): StorageClient public function getObject(string $path): ?StorageObject { if ($this->hasPrefix()) { - $path = implode(DIRECTORY_SEPARATOR, [ + $prefixedPath = implode(DIRECTORY_SEPARATOR, [ $this->getPrefix(), ltrim($path, DIRECTORY_SEPARATOR), ]); } - $object = $this->bucket->object($path); + $object = $this->bucket->object($prefixedPath); if (!$object && $fallback = $this->deploymentConfig->get('storage/fallback_url')) { /* Attempt to load the image from fallback URL and upload to GCS */ @@ -282,9 +282,9 @@ public function getObject(string $path): ?StorageObject $content = curl_exec($ch); curl_close($ch); - if ($content) { + if ($content && curl_getinfo($ch, CURLINFO_HTTP_CODE) === 200) { $object = $this->uploadObject($content, [ - 'name' => $path, + 'name' => $prefixedPath, 'predefinedAcl' => $this->getObjectAclPolicy() ]); } From b2f04ca3f67bb7608db415aff67373853c9c1d00 Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 26 Jan 2022 14:15:08 +0000 Subject: [PATCH 12/62] Improve object exists check --- Model/Adapter/StorageObjectManagement.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 0b9a6b2..bdec771 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -274,13 +274,12 @@ public function getObject(string $path): ?StorageObject $object = $this->bucket->object($prefixedPath); - if (!$object && $fallback = $this->deploymentConfig->get('storage/fallback_url')) { + if (!$object->exists() && $fallback = $this->deploymentConfig->get('storage/fallback_url')) { /* Attempt to load the image from fallback URL and upload to GCS */ $ch = curl_init($fallback . $path); curl_setopt($ch, CURLOPT_TIMEOUT, 5); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); $content = curl_exec($ch); - curl_close($ch); if ($content && curl_getinfo($ch, CURLINFO_HTTP_CODE) === 200) { $object = $this->uploadObject($content, [ @@ -288,6 +287,8 @@ public function getObject(string $path): ?StorageObject 'predefinedAcl' => $this->getObjectAclPolicy() ]); } + + curl_close($ch); } return $object; From 25a7c577db648143454d4f14a53ba1dbb7e7393a Mon Sep 17 00:00:00 2001 From: David Windell Date: Mon, 21 Mar 2022 18:27:37 +0000 Subject: [PATCH 13/62] Further support for loading product images --- Model/File/Storage/Synchronization.php | 8 ++- Plugin/Catalog/View/Asset/Image.php | 71 ++++++++++++++++++++++++++ etc/di.xml | 10 ++++ etc/module.xml | 1 + 4 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 Plugin/Catalog/View/Asset/Image.php diff --git a/Model/File/Storage/Synchronization.php b/Model/File/Storage/Synchronization.php index 1a63b23..ac4d136 100644 --- a/Model/File/Storage/Synchronization.php +++ b/Model/File/Storage/Synchronization.php @@ -37,11 +37,15 @@ public function synchronize($relativeFileName) { /** @var $storage Bucket */ $storage = $this->storageFactory->create(); - + if (!$storage->getStorage()->isEnabled()) { return; } - + + if ($this->mediaDirectory->isFile($relativeFileName)) { + return; + } + try { $storage->loadByFilename($relativeFileName); } catch (\Exception $e) { diff --git a/Plugin/Catalog/View/Asset/Image.php b/Plugin/Catalog/View/Asset/Image.php new file mode 100644 index 0000000..d2c94b4 --- /dev/null +++ b/Plugin/Catalog/View/Asset/Image.php @@ -0,0 +1,71 @@ +mediaDirectory = $filesystem->getDirectoryWrite(DirectoryList::MEDIA); + $this->mediaConfig = $mediaConfig; + $this->storageFactory = $storageFactory; + } + + public function beforeGetUrl(LocalInterface $image) + { + /** @var $storage Storage\Bucket */ + $storage = $this->storageFactory->create(); + + if (!$storage->getStorage()->isEnabled()) { + return; + } + + $relativeFileName = $this->mediaConfig->getBaseMediaPath() . $image->getFilePath(); + + if ($this->mediaDirectory->isFile($relativeFileName)) { + return; + } + + try { + $storage->loadByFilename($relativeFileName); + } catch (\Exception $e) { + } + if ($storage->getId()) { + /** @var WriteInterface $file */ + $file = $this->mediaDirectory->openFile($relativeFileName, 'w'); + try { + $file->lock(); + $file->write($storage->getContent()); + $file->unlock(); + $file->close(); + } catch (FileSystemException $e) { + $file->close(); + } + } + } +} diff --git a/etc/di.xml b/etc/di.xml index e8edb9c..21bfc58 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -115,4 +115,14 @@ type="AuroraExtensions\GoogleCloudStorage\Plugin\Catalog\Product\Image" sortOrder="10"/> + + + + + + diff --git a/etc/module.xml b/etc/module.xml index aa9b072..ecdcc5e 100644 --- a/etc/module.xml +++ b/etc/module.xml @@ -20,6 +20,7 @@ + From 4dea96de59782ff38aa191569a2554c06e5551d0 Mon Sep 17 00:00:00 2001 From: David Windell Date: Mon, 21 Mar 2022 19:15:54 +0000 Subject: [PATCH 14/62] Fix double prefix issue --- Model/Adapter/StorageObjectManagement.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index bdec771..3d665c1 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -340,9 +340,10 @@ public function uploadObject($handle, array $options = []): ?StorageObject $prefix = $this->getPrefix(); if (isset($options['name'])) { + $options['name'] = ltrim($options['name'], DIRECTORY_SEPARATOR); $options['name'] = implode(DIRECTORY_SEPARATOR, [ $prefix, - str_replace(DIRECTORY_SEPARATOR . $prefix . DIRECTORY_SEPARATOR, '', $options['name']), + str_replace($prefix . DIRECTORY_SEPARATOR, '', $options['name']), ]); } else { /** @var StreamInterface $stream */ From 0256e98b8b8c5228c88f256b39beccd21a1515b2 Mon Sep 17 00:00:00 2001 From: David Windell Date: Tue, 22 Mar 2022 19:08:52 +0000 Subject: [PATCH 15/62] Support downloading cached product images --- Model/File/Storage/Bucket.php | 29 +++++++++++++++++++++ Plugin/Catalog/Product/Image.php | 27 +------------------- Plugin/Catalog/View/Asset/Image.php | 39 ++++++++++++++--------------- composer.json | 3 ++- 4 files changed, 51 insertions(+), 47 deletions(-) diff --git a/Model/File/Storage/Bucket.php b/Model/File/Storage/Bucket.php index 56a8994..a439628 100644 --- a/Model/File/Storage/Bucket.php +++ b/Model/File/Storage/Bucket.php @@ -33,6 +33,7 @@ use Magento\Framework\{ App\Filesystem\DirectoryList, Exception\LocalizedException, + Exception\FileSystemException, Filesystem, Filesystem\Driver\File as FileDriver, Model\AbstractModel, @@ -334,6 +335,34 @@ public function saveFile(string $filename) return $this; } + /** + * @param string $filePath + * @return bool + */ + public function downloadFile(string $filePath): bool + { + try { + $this->loadByFilename($filePath); + } catch (\Exception $e) { + return false; + } + + if ($this->getId()) { + $file = $this->filesystem->getDirectoryWrite(DirectoryList::MEDIA)->openFile($filePath, 'w'); + try { + $file->lock(); + $file->write($storage->getContent()); + $file->unlock(); + $file->close(); + return true; + } catch (FileSystemException $e) { + $file->close(); + } + } + + return false; + } + /** * @param string $filePath * @return bool diff --git a/Plugin/Catalog/Product/Image.php b/Plugin/Catalog/Product/Image.php index 91a80df..a46330d 100644 --- a/Plugin/Catalog/Product/Image.php +++ b/Plugin/Catalog/Product/Image.php @@ -5,9 +5,6 @@ use AuroraExtensions\GoogleCloudStorage\Model\File\Storage; use Magento\Catalog\Model\Product\Image as ProductImage; use Magento\Catalog\Model\Product\Media\Config; -use Magento\Framework\Filesystem; -use Magento\Framework\Filesystem\Directory\WriteInterface; -use Magento\Framework\App\Filesystem\DirectoryList; class Image { @@ -16,22 +13,15 @@ class Image */ protected $storageFactory; - /** - * @var WriteInterface - */ - protected $mediaDirectory; - /** * @var Config */ protected $mediaConfig; public function __construct( - Filesystem $filesystem, Config $mediaConfig, Storage\BucketFactory $storageFactory ) { - $this->mediaDirectory = $filesystem->getDirectoryWrite(DirectoryList::MEDIA); $this->mediaConfig = $mediaConfig; $this->storageFactory = $storageFactory; } @@ -51,21 +41,6 @@ public function beforeSetBaseFile(ProductImage $image, $file) return; } - try { - $storage->loadByFilename($relativeFileName); - } catch (\Exception $e) { - } - if ($storage->getId()) { - /** @var WriteInterface $file */ - $file = $this->mediaDirectory->openFile($relativeFileName, 'w'); - try { - $file->lock(); - $file->write($storage->getContent()); - $file->unlock(); - $file->close(); - } catch (FileSystemException $e) { - $file->close(); - } - } + $storage->downloadFile($relativeFileName); } } diff --git a/Plugin/Catalog/View/Asset/Image.php b/Plugin/Catalog/View/Asset/Image.php index d2c94b4..c9a7b3c 100644 --- a/Plugin/Catalog/View/Asset/Image.php +++ b/Plugin/Catalog/View/Asset/Image.php @@ -8,6 +8,7 @@ use Magento\Framework\Filesystem; use Magento\Framework\Filesystem\Directory\WriteInterface; use Magento\Framework\App\Filesystem\DirectoryList; +use OuterEdge\Base\Helper\Image as ImageHelper; class Image { @@ -26,17 +27,24 @@ class Image */ protected $mediaConfig; + /** + * @var ImageHelper + */ + protected $imageHelper; + public function __construct( Filesystem $filesystem, Config $mediaConfig, - Storage\BucketFactory $storageFactory + Storage\BucketFactory $storageFactory, + ImageHelper $imageHelper ) { - $this->mediaDirectory = $filesystem->getDirectoryWrite(DirectoryList::MEDIA); + $this->mediaDirectory = $filesystem->getDirectoryRead(DirectoryList::MEDIA); $this->mediaConfig = $mediaConfig; $this->storageFactory = $storageFactory; + $this->imageHelper = $imageHelper; } - public function beforeGetUrl(LocalInterface $image) + public function afterGetUrl(LocalInterface $image) { /** @var $storage Storage\Bucket */ $storage = $this->storageFactory->create(); @@ -45,26 +53,17 @@ public function beforeGetUrl(LocalInterface $image) return; } + // Download the main image $relativeFileName = $this->mediaConfig->getBaseMediaPath() . $image->getFilePath(); - - if ($this->mediaDirectory->isFile($relativeFileName)) { - return; + if (!$this->mediaDirectory->isFile($relativeFileName)) { + $storage->downloadFile($relativeFileName); } - try { - $storage->loadByFilename($relativeFileName); - } catch (\Exception $e) { - } - if ($storage->getId()) { - /** @var WriteInterface $file */ - $file = $this->mediaDirectory->openFile($relativeFileName, 'w'); - try { - $file->lock(); - $file->write($storage->getContent()); - $file->unlock(); - $file->close(); - } catch (FileSystemException $e) { - $file->close(); + // Download the cached image + if ($image->getModule() == 'cache') { + $cacheFilename = $this->imageHelper->prepareFilename($image->getPath()); + if (!$this->mediaDirectory->isFile($cacheFilename)) { + $storage->downloadFile($cacheFilename); } } } diff --git a/composer.json b/composer.json index 2488825..edcacdb 100644 --- a/composer.json +++ b/composer.json @@ -23,7 +23,8 @@ "magento/framework": "^100||^101||^102||^103", "magento/module-media-storage": "^100||^101", "magento/module-store": "^100||^101", - "auroraextensions/modulecomponents": "~100.0.0||~100.1.0" + "auroraextensions/modulecomponents": "~100.0.0||~100.1.0", + "outeredge/magento-base-module": "^2.6.33" }, "autoload": { "files": [ From add5e504ce07b84062772d00bfc4ad3a1707ae02 Mon Sep 17 00:00:00 2001 From: David Windell Date: Tue, 22 Mar 2022 19:10:26 +0000 Subject: [PATCH 16/62] Remove duplicate code --- Model/File/Storage/Synchronization.php | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/Model/File/Storage/Synchronization.php b/Model/File/Storage/Synchronization.php index ac4d136..d2ce46a 100644 --- a/Model/File/Storage/Synchronization.php +++ b/Model/File/Storage/Synchronization.php @@ -46,21 +46,6 @@ public function synchronize($relativeFileName) return; } - try { - $storage->loadByFilename($relativeFileName); - } catch (\Exception $e) { - } - if ($storage->getId()) { - /** @var WriteInterface $file */ - $file = $this->mediaDirectory->openFile($relativeFileName, 'w'); - try { - $file->lock(); - $file->write($storage->getContent()); - $file->unlock(); - $file->close(); - } catch (FileSystemException $e) { - $file->close(); - } - } + $storage->downloadFile($relativeFileName); } } From 21bf1622baa557703ef4218ea03a207a54955201 Mon Sep 17 00:00:00 2001 From: David Windell Date: Tue, 22 Mar 2022 19:41:52 +0000 Subject: [PATCH 17/62] Fix typo --- Model/File/Storage/Bucket.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Model/File/Storage/Bucket.php b/Model/File/Storage/Bucket.php index a439628..b738355 100644 --- a/Model/File/Storage/Bucket.php +++ b/Model/File/Storage/Bucket.php @@ -351,7 +351,7 @@ public function downloadFile(string $filePath): bool $file = $this->filesystem->getDirectoryWrite(DirectoryList::MEDIA)->openFile($filePath, 'w'); try { $file->lock(); - $file->write($storage->getContent()); + $file->write($this->getContent()); $file->unlock(); $file->close(); return true; From f963ee4f49721a4349cea69d30e65c28ef3bee56 Mon Sep 17 00:00:00 2001 From: David Windell Date: Tue, 22 Mar 2022 20:24:18 +0000 Subject: [PATCH 18/62] Change plugin to before not after --- Plugin/Catalog/View/Asset/Image.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Plugin/Catalog/View/Asset/Image.php b/Plugin/Catalog/View/Asset/Image.php index c9a7b3c..dc77c64 100644 --- a/Plugin/Catalog/View/Asset/Image.php +++ b/Plugin/Catalog/View/Asset/Image.php @@ -44,7 +44,7 @@ public function __construct( $this->imageHelper = $imageHelper; } - public function afterGetUrl(LocalInterface $image) + public function beforeGetUrl(LocalInterface $image) { /** @var $storage Storage\Bucket */ $storage = $this->storageFactory->create(); From 71b0c1e2b17d8a249cf00a0e6de0778f1cfc1d47 Mon Sep 17 00:00:00 2001 From: David Windell Date: Thu, 24 Mar 2022 14:18:22 +0000 Subject: [PATCH 19/62] Category images and some KISS --- Model/File/Storage/Bucket.php | 30 +++++++++++++++----------- Model/File/Storage/Synchronization.php | 18 +--------------- Plugin/Catalog/Product/Image.php | 27 ++++++----------------- Plugin/Catalog/View/Asset/Image.php | 18 ++-------------- env.php.sample | 1 + etc/di.xml | 5 +++++ 6 files changed, 33 insertions(+), 66 deletions(-) diff --git a/Model/File/Storage/Bucket.php b/Model/File/Storage/Bucket.php index b738355..1d9894f 100644 --- a/Model/File/Storage/Bucket.php +++ b/Model/File/Storage/Bucket.php @@ -150,20 +150,14 @@ public function setObjects(?ObjectIterator $objects) } /** - * @param string $filename + * @param string $relativePath * @return $this */ - public function loadByFilename(string $filename) + public function loadByFilename(string $relativePath) { - /** @var string $relativePath */ - $relativePath = $this->storageHelper->getMediaRelativePath($filename); - - // @todo look up media path like https://github.com/magento/magento2/blob/2.4-develop/app/code/Magento/MediaStorage/App/Media.php#L187 - $relativePath = str_replace('media/', '', $relativePath); - if ($this->getStorage()->objectExists($relativePath)) { - $this->setData('id', $filename); - $this->setData('filename', $filename); + $this->setData('id', $relativePath); + $this->setData('filename', $relativePath); $this->setData('content', $this->getStorage()->getObject($relativePath)->downloadAsString()); } else { $this->unsetData(); @@ -337,18 +331,28 @@ public function saveFile(string $filename) /** * @param string $filePath - * @return bool + * @return bool Returns true on existing file or successful download, false on failure */ public function downloadFile(string $filePath): bool { + $relativePath = $this->storageHelper->getMediaRelativePath($filePath); + // @todo look up media path like https://github.com/magento/magento2/blob/2.4-develop/app/code/Magento/MediaStorage/App/Media.php#L187 + $relativePath = str_replace(DirectoryList::MEDIA . DIRECTORY_SEPARATOR, '', $relativePath); + + $mediaPath = $this->filesystem->getDirectoryWrite(DirectoryList::MEDIA); + + if ($mediaPath->isFile($relativePath)) { + return true; + } + try { - $this->loadByFilename($filePath); + $this->loadByFilename($relativePath); } catch (\Exception $e) { return false; } if ($this->getId()) { - $file = $this->filesystem->getDirectoryWrite(DirectoryList::MEDIA)->openFile($filePath, 'w'); + $file = $mediaPath->openFile($relativePath, 'w'); try { $file->lock(); $file->write($this->getContent()); diff --git a/Model/File/Storage/Synchronization.php b/Model/File/Storage/Synchronization.php index d2ce46a..5576a83 100644 --- a/Model/File/Storage/Synchronization.php +++ b/Model/File/Storage/Synchronization.php @@ -2,9 +2,6 @@ namespace AuroraExtensions\GoogleCloudStorage\Model\File\Storage; -use Magento\Framework\Exception\FileSystemException; -use Magento\Framework\Filesystem\Directory\WriteInterface as DirectoryWrite; - class Synchronization { /** @@ -12,19 +9,10 @@ class Synchronization */ protected $storageFactory; - /** - * File stream handler - * - * @var DirectoryWrite - */ - protected $mediaDirectory; - public function __construct( - BucketFactory $storageFactory, - DirectoryWrite $directory + BucketFactory $storageFactory ) { $this->storageFactory = $storageFactory; - $this->mediaDirectory = $directory; } /** @@ -42,10 +30,6 @@ public function synchronize($relativeFileName) return; } - if ($this->mediaDirectory->isFile($relativeFileName)) { - return; - } - $storage->downloadFile($relativeFileName); } } diff --git a/Plugin/Catalog/Product/Image.php b/Plugin/Catalog/Product/Image.php index a46330d..5a68af0 100644 --- a/Plugin/Catalog/Product/Image.php +++ b/Plugin/Catalog/Product/Image.php @@ -2,16 +2,16 @@ namespace AuroraExtensions\GoogleCloudStorage\Plugin\Catalog\Product; -use AuroraExtensions\GoogleCloudStorage\Model\File\Storage; +use AuroraExtensions\GoogleCloudStorage\Model\File\Storage\Synchronization; use Magento\Catalog\Model\Product\Image as ProductImage; use Magento\Catalog\Model\Product\Media\Config; class Image { /** - * @var Storage\BucketFactory + * @var Synchronization */ - protected $storageFactory; + protected $synchronization; /** * @var Config @@ -19,28 +19,15 @@ class Image protected $mediaConfig; public function __construct( - Config $mediaConfig, - Storage\BucketFactory $storageFactory + Synchronization $synchronization, + Config $mediaConfig ) { + $this->synchronization = $synchronization; $this->mediaConfig = $mediaConfig; - $this->storageFactory = $storageFactory; } public function beforeSetBaseFile(ProductImage $image, $file) { - /** @var $storage Storage\Bucket */ - $storage = $this->storageFactory->create(); - - if (!$storage->getStorage()->isEnabled()) { - return; - } - - $relativeFileName = $this->mediaConfig->getBaseMediaPath() . $file; - - if ($this->mediaDirectory->isFile($relativeFileName)) { - return; - } - - $storage->downloadFile($relativeFileName); + $this->synchronization->synchronize($this->mediaConfig->getBaseMediaPath() . $file); } } diff --git a/Plugin/Catalog/View/Asset/Image.php b/Plugin/Catalog/View/Asset/Image.php index dc77c64..540bd5e 100644 --- a/Plugin/Catalog/View/Asset/Image.php +++ b/Plugin/Catalog/View/Asset/Image.php @@ -5,9 +5,6 @@ use Magento\Framework\View\Asset\LocalInterface; use AuroraExtensions\GoogleCloudStorage\Model\File\Storage; use Magento\Catalog\Model\Product\Media\Config; -use Magento\Framework\Filesystem; -use Magento\Framework\Filesystem\Directory\WriteInterface; -use Magento\Framework\App\Filesystem\DirectoryList; use OuterEdge\Base\Helper\Image as ImageHelper; class Image @@ -17,11 +14,6 @@ class Image */ protected $storageFactory; - /** - * @var WriteInterface - */ - protected $mediaDirectory; - /** * @var Config */ @@ -33,12 +25,10 @@ class Image protected $imageHelper; public function __construct( - Filesystem $filesystem, Config $mediaConfig, Storage\BucketFactory $storageFactory, ImageHelper $imageHelper ) { - $this->mediaDirectory = $filesystem->getDirectoryRead(DirectoryList::MEDIA); $this->mediaConfig = $mediaConfig; $this->storageFactory = $storageFactory; $this->imageHelper = $imageHelper; @@ -55,16 +45,12 @@ public function beforeGetUrl(LocalInterface $image) // Download the main image $relativeFileName = $this->mediaConfig->getBaseMediaPath() . $image->getFilePath(); - if (!$this->mediaDirectory->isFile($relativeFileName)) { - $storage->downloadFile($relativeFileName); - } + $storage->downloadFile($relativeFileName); // Download the cached image if ($image->getModule() == 'cache') { $cacheFilename = $this->imageHelper->prepareFilename($image->getPath()); - if (!$this->mediaDirectory->isFile($cacheFilename)) { - $storage->downloadFile($cacheFilename); - } + $storage->downloadFile($cacheFilename); } } } diff --git a/env.php.sample b/env.php.sample index 8c42866..7567747 100644 --- a/env.php.sample +++ b/env.php.sample @@ -10,6 +10,7 @@ return [ 'acl' => 'publicRead', 'region' => 'us-east1', ], + 'fallback_url' => 'https://example.com/media/' ], ], ]; diff --git a/etc/di.xml b/etc/di.xml index 21bfc58..bc4d8c3 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -125,4 +125,9 @@ type="AuroraExtensions\GoogleCloudStorage\Plugin\Catalog\View\Asset\Image" sortOrder="10"/> + + + From aef75c3ecd8101b0b9bcff07419c0a37cf5d78f3 Mon Sep 17 00:00:00 2001 From: David Windell Date: Thu, 24 Mar 2022 16:32:31 +0000 Subject: [PATCH 20/62] Add Category plugin --- Plugin/Catalog/Category.php | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 Plugin/Catalog/Category.php diff --git a/Plugin/Catalog/Category.php b/Plugin/Catalog/Category.php new file mode 100644 index 0000000..8f21524 --- /dev/null +++ b/Plugin/Catalog/Category.php @@ -0,0 +1,26 @@ +synchronization = $synchronization; + } + + public function afterGetImageUrl(CategoryModel $category, $result) + { + $this->synchronization->synchronize($result); + return $result; + } +} From 53444ed34f027cbec2481527abb1889518e406a3 Mon Sep 17 00:00:00 2001 From: David Windell Date: Mon, 23 May 2022 17:48:54 +0100 Subject: [PATCH 21/62] Bump dependency versions --- composer.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/composer.json b/composer.json index edcacdb..6fe9cce 100644 --- a/composer.json +++ b/composer.json @@ -13,18 +13,18 @@ "email": "support@auroraextensions.com" }, "require": { - "php": "~7.2.0||~7.3.0||~7.4.0", + "php": "^7.2|^8.0", "psr/http-message": "~1.0", "psr/log": "~1.0", "symfony/console": "^2.7||^3.0||^4.0||^5.0", "google/crc32": "~0.1.0", "google/cloud-core": "~1.33.0", "google/cloud-storage": "~1.14.0", - "magento/framework": "^100||^101||^102||^103", + "magento/framework": "^100||^101||^102||^103||^104", "magento/module-media-storage": "^100||^101", "magento/module-store": "^100||^101", "auroraextensions/modulecomponents": "~100.0.0||~100.1.0", - "outeredge/magento-base-module": "^2.6.33" + "outeredge/magento-base-module": "^2.6.33||^3.0" }, "autoload": { "files": [ From 836662d915c0981d2d0ecee162ed3af85a466368 Mon Sep 17 00:00:00 2001 From: David Windell Date: Mon, 23 May 2022 17:58:18 +0000 Subject: [PATCH 22/62] Remove dependency on ModuleComponents (for PHP8) --- .gitignore | 1 - ...ocalizedScopeDeploymentConfigInterface.php | 28 +++ Console/Command/SynchronizeCommand.php | 121 ----------- Exception/ExceptionFactory.php | 84 ++++++++ Model/Adapter/StorageObjectManagement.php | 4 +- Model/Config/Source/Select/VirtualSelect.php | 71 +++++++ Model/File/Storage/Bucket.php | 4 +- Model/Utils/PathUtils.php | 201 ++++++++++++++++++ .../File/Uploader/ObjectUploader.php | 4 +- README.rst | 85 -------- composer.json | 22 +- etc/module.xml | 1 - 12 files changed, 393 insertions(+), 233 deletions(-) delete mode 100644 .gitignore create mode 100644 Api/LocalizedScopeDeploymentConfigInterface.php delete mode 100644 Console/Command/SynchronizeCommand.php create mode 100644 Exception/ExceptionFactory.php create mode 100644 Model/Config/Source/Select/VirtualSelect.php create mode 100644 Model/Utils/PathUtils.php delete mode 100644 README.rst diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 4c933b8..0000000 --- a/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.sw[po] diff --git a/Api/LocalizedScopeDeploymentConfigInterface.php b/Api/LocalizedScopeDeploymentConfigInterface.php new file mode 100644 index 0000000..f372966 --- /dev/null +++ b/Api/LocalizedScopeDeploymentConfigInterface.php @@ -0,0 +1,28 @@ + + * @license MIT + */ +declare(strict_types=1); + +namespace AuroraExtensions\GoogleCloudStorage\Api; + +interface LocalizedScopeDeploymentConfigInterface +{ + /** + * @param string|null $path + * @return mixed + */ + public function get(?string $path); +} \ No newline at end of file diff --git a/Console/Command/SynchronizeCommand.php b/Console/Command/SynchronizeCommand.php deleted file mode 100644 index 7749022..0000000 --- a/Console/Command/SynchronizeCommand.php +++ /dev/null @@ -1,121 +0,0 @@ - - * @license MIT - */ -declare(strict_types=1); - -namespace AuroraExtensions\GoogleCloudStorage\Console\Command; - -use Exception; -use AuroraExtensions\GoogleCloudStorage\Api\StorageTypeMetadataInterface; -use Magento\Framework\{ - App\Area, - App\State -}; -use Magento\MediaStorage\{ - Model\File\Storage, - Model\File\Storage\Flag -}; -use Psr\Log\LoggerInterface; -use Symfony\Component\{ - Console\Command\Command, - Console\Input\InputInterface, - Console\Output\OutputInterface -}; - -use const null; -use function strtotime; -use function time; - -class SynchronizeCommand extends Command -{ - /** @constant string COMMAND_NAME */ - private const COMMAND_NAME = 'gcs:media:sync'; - - /** @constant string COMMAND_DESC */ - private const COMMAND_DESC = 'Synchronize media storage with Google Cloud Storage.'; - - /** @var State $state */ - private $state; - - /** @var Storage $storage */ - private $storage; - - /** @var LoggerInterface $logger */ - private $logger; - - /** - * @param State $state - * @param Storage $storage - * @param LoggerInterface $logger - * @return void - */ - public function __construct( - State $state, - Storage $storage, - LoggerInterface $logger - ) { - $this->state = $state; - $this->storage = $storage; - $this->logger = $logger; - parent::__construct(); - } - - /** - * {@inheritdoc} - */ - protected function configure() - { - $this->setName(self::COMMAND_NAME); - $this->setDescription(self::COMMAND_DESC); - parent::configure(); - } - - /** - * {@inheritdoc} - */ - protected function execute(InputInterface $input, OutputInterface $output) - { - try { - $this->state->setAreaCode(Area::AREA_ADMINHTML); - - /** @var Flag $flag */ - $flag = $this->storage->getSyncFlag(); - - /** @var int|string|null $lastUpdate */ - $lastUpdate = $flag->getLastUpdate() ?: null; - - if ($flag->getState() === Flag::STATE_RUNNING && !empty($lastUpdate) && time() <= strtotime($lastUpdate) + Flag::FLAG_TTL) { - return; - } - - $flag->setState(Flag::STATE_RUNNING)->setFlagData([])->save(); - - try { - $this->storage->synchronize([ - 'type' => StorageTypeMetadataInterface::STORAGE_MEDIA_GCS, - ]); - } catch (Exception $e) { - $this->logger->critical($e); - $flag->passError($e); - } - - $flag->setState(Flag::STATE_FINISHED)->save(); - $output->writeln('Media synchronized successfully!'); - } catch (Exception $e) { - $output->writeln('Media synchronization failed!'); - } - } -} diff --git a/Exception/ExceptionFactory.php b/Exception/ExceptionFactory.php new file mode 100644 index 0000000..43fb6b5 --- /dev/null +++ b/Exception/ExceptionFactory.php @@ -0,0 +1,84 @@ + + * @license MIT + */ +declare(strict_types=1); + +namespace AuroraExtensions\GoogleCloudStorage\Exception; + +use Exception; +use Throwable; +use Magento\Framework\{ + ObjectManagerInterface, + Phrase +}; + +use function is_subclass_of; +use function __; + +class ExceptionFactory +{ + /** @constant string ERROR_DEFAULT_MSG */ + private const ERROR_DEFAULT_MSG = 'An error occurred. Unable to process the request.'; + + /** @constant string ERROR_INVALID_TYPE */ + private const ERROR_INVALID_TYPE = 'Invalid exception class type %1 was given.'; + + /** @var ObjectManagerInterface $objectManager */ + private $objectManager; + + /** + * @param ObjectManagerInterface $objectManager + * @return void + */ + public function __construct(ObjectManagerInterface $objectManager) + { + $this->objectManager = $objectManager; + } + + /** + * @param string|null $type + * @param Phrase|null $message + * @return Throwable + * @throws Exception + */ + public function create( + string $type = Exception::class, + Phrase $message = null + ) { + /** @var array $arguments */ + $arguments = []; + + /* Set default message, as required. */ + $message = $message ?? __(static::ERROR_DEFAULT_MSG); + + if (!is_subclass_of($type, Throwable::class)) { + throw new Exception( + __( + static::ERROR_INVALID_TYPE, + $type + )->__toString() + ); + } + + if ($type !== Exception::class) { + $arguments['phrase'] = $message; + } else { + $arguments['message'] = $message->__toString(); + } + + return $this->objectManager->create($type, $arguments); + } +} \ No newline at end of file diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 3d665c1..516479c 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -24,9 +24,7 @@ Component\ModuleConfigTrait, Exception\InvalidGoogleCloudStorageSetupException, Model\System\ModuleConfig, - Model\File\Storage -}; -use AuroraExtensions\ModuleComponents\{ + Model\File\Storage, Api\LocalizedScopeDeploymentConfigInterface, Api\LocalizedScopeDeploymentConfigInterfaceFactory, Exception\ExceptionFactory diff --git a/Model/Config/Source/Select/VirtualSelect.php b/Model/Config/Source/Select/VirtualSelect.php new file mode 100644 index 0000000..7bd3ebb --- /dev/null +++ b/Model/Config/Source/Select/VirtualSelect.php @@ -0,0 +1,71 @@ + + * @license MIT + */ +declare(strict_types=1); + +namespace AuroraExtensions\GoogleCloudStorage\Model\Config\Source\Select; + +use Magento\Framework\Data\OptionSourceInterface; + +use const true; +use function array_flip; +use function array_walk; +use function __; + +class VirtualSelect implements OptionSourceInterface +{ + /** @var array $options */ + private $options = []; + + /** + * @param array $data + * @param bool $flip + * @return void + */ + public function __construct( + array $data = [], + bool $flip = true + ) { + /** @var array $opts */ + $opts = $flip ? array_flip($data) : $data; + + array_walk($opts, [ + $this, + 'setOption' + ]); + } + + /** + * @param int|string|null $value + * @param int|string $key + * @return void + */ + private function setOption($value, $key): void + { + $this->options[] = [ + 'label' => __($key), + 'value' => $value, + ]; + } + + /** + * @return array + */ + public function toOptionArray() + { + return $this->options; + } +} \ No newline at end of file diff --git a/Model/File/Storage/Bucket.php b/Model/File/Storage/Bucket.php index 1d9894f..7228614 100644 --- a/Model/File/Storage/Bucket.php +++ b/Model/File/Storage/Bucket.php @@ -23,9 +23,9 @@ Api\StorageObjectManagementInterface, Component\ModuleConfigTrait, Component\StorageAdapterTrait, - Model\System\ModuleConfig + Model\System\ModuleConfig, + Exception\ExceptionFactory }; -use AuroraExtensions\ModuleComponents\Exception\ExceptionFactory; use Google\Cloud\{ Storage\StorageObject, Storage\ObjectIterator diff --git a/Model/Utils/PathUtils.php b/Model/Utils/PathUtils.php new file mode 100644 index 0000000..813c717 --- /dev/null +++ b/Model/Utils/PathUtils.php @@ -0,0 +1,201 @@ + + * @license MIT + */ +declare(strict_types=1); + +namespace AuroraExtensions\GoogleCloudStorage\Model\Utils; + +use Magento\Framework\Stdlib\StringUtils; + +use const DIRECTORY_SEPARATOR; +use const PHP_MAXPATHLEN; +use function array_filter; +use function array_map; +use function array_merge; +use function array_pop; +use function array_shift; +use function array_values; +use function explode; +use function implode; + +class PathUtils +{ + /** @constant int TRIM_LEFT */ + public const TRIM_LEFT = 1; + + /** @constant int TRIM_RIGHT */ + public const TRIM_RIGHT = 2; + + /** @constant array TRIM_CONTEXTS */ + public const TRIM_CONTEXTS = [ + self::TRIM_LEFT => 'ltrim', + self::TRIM_RIGHT => 'rtrim', + self::TRIM_LEFT | self::TRIM_RIGHT => 'trim', + ]; + + /** @var StringUtils $stringUtils */ + private $stringUtils; + + /** + * @param StringUtils $stringUtils + * @return void + */ + public function __construct(StringUtils $stringUtils) + { + $this->stringUtils = $stringUtils; + } + + /** + * @param string[] $pieces + * @return string + */ + public function build(string ...$pieces): string + { + /** @var string $basePath */ + $basePath = $this->trim((string) array_shift($pieces), self::TRIM_RIGHT); + + if (!empty($basePath)) { + /** @var bool $isAbsolute */ + $isAbsolute = false; + + if ($basePath[0] === DIRECTORY_SEPARATOR) { + $isAbsolute = true; + } + + /** @var string[] $baseParts */ + $baseParts = $this->stringUtils->split($basePath, PHP_MAXPATHLEN, true, true, '\/'); + + /** @var string[] $basePaths */ + $basePaths = $this->split(!empty($baseParts) ? $baseParts[0] : $basePath, ' ', true); + $basePath = $this->concat($isAbsolute ? array_merge([null], $basePaths) : $basePaths); + } + + /** @var array $result */ + $result[] = [$basePath]; + + /** @var string $basename */ + $basename = $this->trim((string) array_pop($pieces)); + + if (!empty($basename)) { + /** @var string[] $nameParts */ + $nameParts = $this->split($basename, DIRECTORY_SEPARATOR, true); + $basename = $this->concat($nameParts); + } + + /** @var string[] $dirs */ + $dirs = $this->filter($pieces); + + /** @var int|string $key */ + /** @var string $value */ + foreach ($dirs as $key => $value) { + /** @var string[] $parts */ + $parts = $this->stringUtils->split($value, PHP_MAXPATHLEN, true, true, '\/'); + + /** @var string[] $paths */ + $paths = $this->split(!empty($parts) ? $parts[0] : $value, ' ', true); + + /** @var string $path */ + $path = $this->concat($paths); + $dirs[$key] = $this->trim($path); + } + + $result[] = $dirs; + $result[] = [$basename]; + + return $this->concat(array_merge(...$result)); + } + + /** + * @param array $pieces + * @param string $delimiter + * @param bool $filter + * @return string + */ + public function concat( + array $pieces, + string $delimiter = DIRECTORY_SEPARATOR, + bool $filter = false + ): string { + if ($filter) { + $pieces = array_filter($pieces, 'strlen'); + } + + return implode($delimiter, $pieces); + } + + /** + * @param array $pieces + * @param callable|null $callback + * @param bool $preserveKeys + * @return array + */ + public function filter( + array $pieces, + callable $callback = null, + bool $preserveKeys = false + ): array { + /* Defaults strlen for empty values. */ + $callback = $callback ?? 'strlen'; + + /** @var array $result */ + $result = array_filter($pieces, $callback); + return $preserveKeys ? $result : array_values($result); + } + + /** + * @param string $subject + * @param string $delimiter + * @param bool $filter + * @return string[] + */ + public function split( + string $subject, + string $delimiter = DIRECTORY_SEPARATOR, + bool $filter = false + ): array { + /** @var string[] $pieces */ + $pieces = explode($delimiter, $subject); + + if ($filter) { + $pieces = array_filter($pieces, 'strlen'); + } + + return array_values($pieces); + } + + /** + * @param string $subject + * @param int $context + * @param string $delimiter + * @return string + */ + public function trim( + string $subject, + int $context = self::TRIM_LEFT | self::TRIM_RIGHT, + string $delimiter = DIRECTORY_SEPARATOR + ): string { + /** @var string|null $callback */ + $callback = self::TRIM_CONTEXTS[$context] ?? null; + + if ($callback === null) { + return $subject; + } + + /** @var array $result */ + $result = array_map($callback, [$subject], [$delimiter]); + return !empty($result) ? $result[0] : $subject; + } +} \ No newline at end of file diff --git a/Plugin/Framework/File/Uploader/ObjectUploader.php b/Plugin/Framework/File/Uploader/ObjectUploader.php index 1714074..9f75bcb 100644 --- a/Plugin/Framework/File/Uploader/ObjectUploader.php +++ b/Plugin/Framework/File/Uploader/ObjectUploader.php @@ -23,9 +23,9 @@ Api\StorageObjectManagementInterface, Component\ModuleConfigTrait, Component\StorageAdapterTrait, - Model\System\ModuleConfig + Model\System\ModuleConfig, + Model\Utils\PathUtils }; -use AuroraExtensions\ModuleComponents\Model\Utils\PathUtils; use Magento\Framework\{ Exception\FileSystemException, File\Uploader, diff --git a/README.rst b/README.rst deleted file mode 100644 index c95e41f..0000000 --- a/README.rst +++ /dev/null @@ -1,85 +0,0 @@ -Google Cloud Storage -==================== - -.. contents:: - :local: - -Description ------------ - -.. |link1| replace:: Google Cloud Storage -.. |link2| replace:: Google Cloud CDN -.. |link3| replace:: Documentation -.. |link4| replace:: Creating and Managing Service Account Keys -.. |link5| replace:: env.php.sample -.. _link1: https://cloud.google.com/storage/ -.. _link2: https://cloud.google.com/cdn/ -.. _link3: https://docs.auroraextensions.com/magento/extensions/2.x/googlecloudstorage/latest/index.html -.. _link4: https://cloud.google.com/iam/docs/creating-managing-service-account-keys -.. _link5: https://github.com/auroraextensions/googlecloudstorage/blob/master/env.php.sample - -Use |link1|_ to store media assets in Magento. - -Installation ------------- - -We highly recommend installing via Composer for package management. - -.. code-block:: sh - - composer require auroraextensions/googlecloudstorage - -Configuration -------------- - -Once installed, update the environment configuration file. See |link5|_ for an example. -The following information should be readily available: - -1. Google Cloud project ID -2. Path to the Google Cloud service account JSON key file. See `Service Account`_ for more details. -3. Google Cloud Storage bucket name -4. Google Cloud Storage bucket region (if applicable) - -Next, enable the module with the Magento autoloader. - -.. code-block:: sh - - php bin/magento module:enable AuroraExtensions_GoogleCloudStorage - -Synchronization ---------------- - -You can initiate the bulk synchronization process through the Magento backend, just as you would with -any other media storage configuration. Additionally, you can initiate the bulk synchronization process -from the command line using the provided synchronization CLI command. - -.. code-block:: sh - - php bin/magento gcs:media:sync - -**IMPORTANT**: This process can be very slow, especially if you have a lot of media files. - -Service Account ---------------- - -For the purposes of authenticating with Google Cloud Platform, this module leverages the flexibility and ease of use provided by Google Cloud service accounts. -Before moving forward, please make sure to complete the following: - -1. Create a Google Cloud service account with **Storage Admin** privileges. Once the service account is created, you will be prompted to download a JSON key file. Store this key file in a safe place. -2. Install the service account JSON key file to the local or mounted filesystem with read-only permissions for the Magento user. -3. Verify the following fields are set and correct in the environment configuration file (env.php): - 1. All required fields - 2. The Google Cloud project name is where the bucket exists - 3. The path to the Google Cloud service account JSON key file (e.g. /etc/gcs.json). Relative paths are assumed to be relative to the Magento root directory. - 4. [OPTIONAL] If you use the same bucket for multiple projects, you can specify a subdirectory to synchronize to inside the bucket. By default, it will synchronize to /. - -For more information on Google Cloud service account keys, please see |link4|_. - -For an example configuration file, please see |link5|_. - -Troubleshooting ---------------- - - Given keyfile at path /path/to/magento was invalid - -You need to create and install a service account key to authenticate with Google Cloud. See `Service Account`_ for specific details on Google Cloud service accounts. diff --git a/composer.json b/composer.json index 6fe9cce..f96530d 100644 --- a/composer.json +++ b/composer.json @@ -3,27 +3,13 @@ "description": "Google Cloud Storage integration for Magento.", "type": "magento2-module", "license": "MIT", - "authors": [ - { - "name": "Nickolas Burr", - "email": "nickolasburr@auroraextensions.com" - } - ], - "support": { - "email": "support@auroraextensions.com" - }, "require": { - "php": "^7.2|^8.0", "psr/http-message": "~1.0", "psr/log": "~1.0", - "symfony/console": "^2.7||^3.0||^4.0||^5.0", - "google/crc32": "~0.1.0", - "google/cloud-core": "~1.33.0", - "google/cloud-storage": "~1.14.0", - "magento/framework": "^100||^101||^102||^103||^104", - "magento/module-media-storage": "^100||^101", - "magento/module-store": "^100||^101", - "auroraextensions/modulecomponents": "~100.0.0||~100.1.0", + "google/cloud-storage": "~1.27.0", + "magento/framework": "^103", + "magento/module-media-storage": "^101", + "magento/module-store": "^101", "outeredge/magento-base-module": "^2.6.33||^3.0" }, "autoload": { diff --git a/etc/module.xml b/etc/module.xml index ecdcc5e..d541b7b 100644 --- a/etc/module.xml +++ b/etc/module.xml @@ -19,7 +19,6 @@ - From 7d1f6eec11724a49e7844ea992f5db4448a408d6 Mon Sep 17 00:00:00 2001 From: David Windell Date: Mon, 23 May 2022 19:06:09 +0100 Subject: [PATCH 23/62] Update composer.json --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index f96530d..946183d 100644 --- a/composer.json +++ b/composer.json @@ -8,7 +8,7 @@ "psr/log": "~1.0", "google/cloud-storage": "~1.27.0", "magento/framework": "^103", - "magento/module-media-storage": "^101", + "magento/module-media-storage": "^100", "magento/module-store": "^101", "outeredge/magento-base-module": "^2.6.33||^3.0" }, From ccae2cab588e7b142ab857b4274902d8d4943f00 Mon Sep 17 00:00:00 2001 From: David Windell Date: Mon, 23 May 2022 18:12:52 +0000 Subject: [PATCH 24/62] Remove references to Console --- etc/di.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/etc/di.xml b/etc/di.xml index bc4d8c3..b8adbf0 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -63,13 +63,6 @@ - - - - AuroraExtensions\GoogleCloudStorage\Console\Command\SynchronizeCommand - - - Date: Tue, 24 May 2022 10:31:50 +0000 Subject: [PATCH 25/62] Add missing component file --- .../LocalizedScopeDeploymentConfig.php | 78 +++++++++++++++++++ etc/di.xml | 7 +- 2 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 Model/Config/Deployment/LocalizedScopeDeploymentConfig.php diff --git a/Model/Config/Deployment/LocalizedScopeDeploymentConfig.php b/Model/Config/Deployment/LocalizedScopeDeploymentConfig.php new file mode 100644 index 0000000..8859e98 --- /dev/null +++ b/Model/Config/Deployment/LocalizedScopeDeploymentConfig.php @@ -0,0 +1,78 @@ + + * @license MIT + */ +declare(strict_types=1); + +namespace AuroraExtensions\GoogleCloudStorage\Model\Config\Deployment; + +use AuroraExtensions\GoogleCloudStorage\Api\LocalizedScopeDeploymentConfigInterface; +use Magento\Framework\App\DeploymentConfig; + +use function array_filter; +use function array_merge; +use function explode; +use function implode; +use function trim; + +class LocalizedScopeDeploymentConfig implements LocalizedScopeDeploymentConfigInterface +{ + /** @constant string DELIMITER */ + private const DELIMITER = '/'; + + /** @var DeploymentConfig $deploymentConfig */ + private $deploymentConfig; + + /** @var string $delimiter */ + private $delimiter; + + /** @var string $scope */ + private $scope; + + /** + * @param DeploymentConfig $deploymentConfig + * @param string $delimiter + * @param string|null $scope + * @return void + */ + public function __construct( + DeploymentConfig $deploymentConfig, + string $delimiter = self::DELIMITER, + string $scope = null + ) { + $this->deploymentConfig = $deploymentConfig; + $this->delimiter = $delimiter; + $this->scope = $scope ?? ''; + } + + /** + * {@inheritdoc} + */ + public function get(?string $path = null) + { + /** @var string $scope */ + $scope = trim($this->scope, $this->delimiter); + + /** @var array $parts */ + $parts = explode($this->delimiter, !empty($path) ? $path : ''); + + /** @var array $merge */ + $merge = array_merge([$scope], array_filter($parts, 'strlen')); + + /** @var string $xpath */ + $xpath = implode($this->delimiter, $merge); + return !empty($xpath) ? $this->deploymentConfig->get($xpath) : null; + } +} diff --git a/etc/di.xml b/etc/di.xml index b8adbf0..d3241d4 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -17,6 +17,9 @@ */ --> + + @@ -24,7 +27,7 @@ type="AuroraExtensions\GoogleCloudStorage\Model\File\Storage\Synchronization"/> + type="AuroraExtensions\GoogleCloudStorage\Model\Config\Source\Select\VirtualSelect"> Authenticated Read @@ -38,7 +41,7 @@ + type="AuroraExtensions\GoogleCloudStorage\Model\Config\Source\Select\VirtualSelect"> Multi-region (multi-region) From 4eb350ca84271ffdd6e399cb15b0bf6115c91b76 Mon Sep 17 00:00:00 2001 From: David Windell Date: Fri, 27 May 2022 19:18:51 +0100 Subject: [PATCH 26/62] Update composer.json --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 946183d..cddb008 100644 --- a/composer.json +++ b/composer.json @@ -10,7 +10,7 @@ "magento/framework": "^103", "magento/module-media-storage": "^100", "magento/module-store": "^101", - "outeredge/magento-base-module": "^2.6.33||^3.0" + "outeredge/magento-base-module": ">=2.6.33" }, "autoload": { "files": [ From 186febf7f77c273026268805b4e2471662e51696 Mon Sep 17 00:00:00 2001 From: David Windell Date: Mon, 30 May 2022 14:40:40 +0100 Subject: [PATCH 27/62] Add image fallback listener --- Block/ImageCatcher.php | 53 +++++++++++++++++++++ view/frontend/layout/default.xml | 8 ++++ view/frontend/templates/image-catcher.phtml | 20 ++++++++ 3 files changed, 81 insertions(+) create mode 100644 Block/ImageCatcher.php create mode 100644 view/frontend/layout/default.xml create mode 100644 view/frontend/templates/image-catcher.phtml diff --git a/Block/ImageCatcher.php b/Block/ImageCatcher.php new file mode 100644 index 0000000..a930da6 --- /dev/null +++ b/Block/ImageCatcher.php @@ -0,0 +1,53 @@ +storeManager = $storeManager; + $this->appState = $appState; + } + + public function getMediaUrl() + { + return $this->storeManager->getStore()->getBaseUrl(UrlInterface::URL_TYPE_MEDIA); + } + + public function getBaseMediaUrl() + { + return $this->storeManager->getStore()->getBaseUrl(UrlInterface::URL_TYPE_WEB) . DirectoryList::MEDIA . DIRECTORY_SEPARATOR; + } + + public function isDeveloperMode() + { + return $this->appState->getMode() == State::MODE_DEVELOPER; + } +} diff --git a/view/frontend/layout/default.xml b/view/frontend/layout/default.xml new file mode 100644 index 0000000..fecc883 --- /dev/null +++ b/view/frontend/layout/default.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/view/frontend/templates/image-catcher.phtml b/view/frontend/templates/image-catcher.phtml new file mode 100644 index 0000000..5e4fb07 --- /dev/null +++ b/view/frontend/templates/image-catcher.phtml @@ -0,0 +1,20 @@ +isDeveloperMode()): ?> + + From bccf5bb4b908095e0355a1ac61aa05e02ff04c7e Mon Sep 17 00:00:00 2001 From: David Windell Date: Tue, 28 Jun 2022 17:33:25 +0000 Subject: [PATCH 28/62] Always run ImageCatcher when GCS is enabled --- Block/ImageCatcher.php | 11 ++++++----- view/frontend/templates/image-catcher.phtml | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Block/ImageCatcher.php b/Block/ImageCatcher.php index a930da6..8e38eab 100644 --- a/Block/ImageCatcher.php +++ b/Block/ImageCatcher.php @@ -2,6 +2,7 @@ namespace AuroraExtensions\GoogleCloudStorage\Block; +use AuroraExtensions\GoogleCloudStorage\Model\File\Storage; use Magento\Framework\App\Filesystem\DirectoryList; use Magento\Framework\App\State; use Magento\Framework\UrlInterface; @@ -21,19 +22,19 @@ class ImageCatcher extends Template /** * @param Context $context * @param Config $mediaConfig - * @param State $appState + * @param Storage $storage * @param array $data */ public function __construct( Context $context, StoreManagerInterface $storeManager, - State $appState, + Storage $appState, array $data = [] ) { parent::__construct($context, $data); $this->storeManager = $storeManager; - $this->appState = $appState; + $this->storage = $storage; } public function getMediaUrl() @@ -46,8 +47,8 @@ public function getBaseMediaUrl() return $this->storeManager->getStore()->getBaseUrl(UrlInterface::URL_TYPE_WEB) . DirectoryList::MEDIA . DIRECTORY_SEPARATOR; } - public function isDeveloperMode() + public function isEnabled() { - return $this->appState->getMode() == State::MODE_DEVELOPER; + return $this->storage->checkBucketUsage(); } } diff --git a/view/frontend/templates/image-catcher.phtml b/view/frontend/templates/image-catcher.phtml index 5e4fb07..9f27dff 100644 --- a/view/frontend/templates/image-catcher.phtml +++ b/view/frontend/templates/image-catcher.phtml @@ -1,4 +1,4 @@ -isDeveloperMode()): ?> +isEnabled()): ?> From 00d1e53ea2d7448724bfb67e09ef04c635f9cd30 Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 16 Nov 2022 21:27:20 +0000 Subject: [PATCH 36/62] Check if file exists before trying to download --- Model/Adapter/StorageObjectManagement.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 98e9fab..c99e87a 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -341,6 +341,11 @@ public function getObjects(array $options = []): ?ObjectIterator */ public function objectExists(string $path): bool { + // Don't waste requests if path does not have an extension + if (strpos($path, '.') === false) { + return false; + } + /** @var StorageObject|null $object */ $object = $this->getObject($path); return ($object !== null && $object->exists()); From c0339f12455f15a07393d345e2eca0c932aee892 Mon Sep 17 00:00:00 2001 From: David Windell Date: Tue, 29 Nov 2022 11:57:40 +0000 Subject: [PATCH 37/62] Remove console.log --- view/frontend/templates/image-catcher.phtml | 1 - 1 file changed, 1 deletion(-) diff --git a/view/frontend/templates/image-catcher.phtml b/view/frontend/templates/image-catcher.phtml index 287d376..fa99da9 100644 --- a/view/frontend/templates/image-catcher.phtml +++ b/view/frontend/templates/image-catcher.phtml @@ -34,7 +34,6 @@ $('[data-gallery-role=gallery-placeholder]').on('gallery:loaded', async function () { const api = $(this).data('gallery'); const images = $('[data-gallery-role=gallery-placeholder]').data('gallery').fotorama.data; - console.log(images) const checkIfImageExists = (url, callback) => { const img = new Image(); From 960bbd503367369fbffb53e739ea1125e4528f64 Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 7 Dec 2022 12:24:04 +0000 Subject: [PATCH 38/62] Wait until dom loaded to fire callback --- view/frontend/templates/image-catcher.phtml | 74 +++++++++++---------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/view/frontend/templates/image-catcher.phtml b/view/frontend/templates/image-catcher.phtml index fa99da9..7f91daf 100644 --- a/view/frontend/templates/image-catcher.phtml +++ b/view/frontend/templates/image-catcher.phtml @@ -5,7 +5,7 @@ var replacedImages = []; - document.addEventListener('error', (event) => { + window.addEventListener('error', (event) => { if (event.target.tagName.toLowerCase() !== 'img') return; const url = event.target.src.split('?')[0]; if (replacedImages.includes(url)) return; @@ -27,49 +27,51 @@ passive: true }); - // Only run the mage/gallery require on product page - if (document.body.classList.contains('catalog-product-view')) { - require(['jquery', 'mage/gallery/gallery'], function($, gallery){ + window.addEventListener('load', function () { + // Only run the mage/gallery require on product page + if (document.body.classList.contains('catalog-product-view')) { + require(['jquery', 'mage/gallery/gallery'], function($, gallery){ - $('[data-gallery-role=gallery-placeholder]').on('gallery:loaded', async function () { - const api = $(this).data('gallery'); - const images = $('[data-gallery-role=gallery-placeholder]').data('gallery').fotorama.data; + $('[data-gallery-role=gallery-placeholder]').on('gallery:loaded', async function () { + const api = $(this).data('gallery'); + const images = $('[data-gallery-role=gallery-placeholder]').data('gallery').fotorama.data; - const checkIfImageExists = (url, callback) => { - const img = new Image(); - img.src = url; + const checkIfImageExists = (url, callback) => { + const img = new Image(); + img.src = url; - if (img.complete) { - callback(true); - } else { - img.onload = () => { + if (img.complete) { callback(true); - }; + } else { + img.onload = () => { + callback(true); + }; - img.onerror = () => { - callback(false); - }; + img.onerror = () => { + callback(false); + }; + } } - } - let replacedGalleryImages = []; - const checkAllImages = () => { - images.forEach(image => { - checkIfImageExists(image.img, (exists) => { - if (!exists) { - image.img = image.img.replace(cdnUrl, mediaUrl) + '?imgstore=getStoreCode() ?>'; - image.thumb = image.img; - } - replacedGalleryImages.push(image); + let replacedGalleryImages = []; + const checkAllImages = () => { + images.forEach(image => { + checkIfImageExists(image.img, (exists) => { + if (!exists) { + image.img = image.img.replace(cdnUrl, mediaUrl) + '?imgstore=getStoreCode() ?>'; + image.thumb = image.img; + } + replacedGalleryImages.push(image); + }); }); - }); - } - await checkAllImages(); - setTimeout(() => { - api.updateData(replacedGalleryImages) - }, 5000); + } + await checkAllImages(); + setTimeout(() => { + api.updateData(replacedGalleryImages) + }, 5000); + }); }); - }); - } + } + }); From a6286c5ae5dff063318e42a95d2157d100b56830 Mon Sep 17 00:00:00 2001 From: Bruno Dias Date: Wed, 19 Jul 2023 15:18:48 +0100 Subject: [PATCH 39/62] Cache gcs (#2) * Added GCS cache --------- Co-authored-by: David Windell --- Model/Adapter/StorageObjectManagement.php | 30 +++++++++++++++++++++++ Model/Cache/Type/GcsCache.php | 24 ++++++++++++++++++ etc/cache.xml | 7 ++++++ 3 files changed, 61 insertions(+) create mode 100644 Model/Cache/Type/GcsCache.php create mode 100644 etc/cache.xml diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index c99e87a..ac5f2b0 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -41,6 +41,9 @@ Filesystem\Driver\File as FileDriver }; use Magento\Store\Model\StoreManagerInterface; +use Magento\Framework\App\CacheInterface; +use Magento\Framework\Serialize\SerializerInterface; +use AuroraExtensions\GoogleCloudStorage\Model\Cache\Type\GcsCache; use Psr\Http\{ Message\StreamInterface, Message\StreamInterfaceFactory @@ -98,6 +101,12 @@ class StorageObjectManagement implements StorageObjectManagementInterface, Stora /** @var StoreManagerInterface $storeManager */ private $storeManager; + /** @var CacheInterface $cache */ + private $cache; + + /** @var SerializerInterface $serializer */ + private $serializer; + /** * @param LocalizedScopeDeploymentConfigInterfaceFactory $deploymentConfigFactory * @param ExceptionFactory $exceptionFactory @@ -105,6 +114,8 @@ class StorageObjectManagement implements StorageObjectManagementInterface, Stora * @param Filesystem $filesystem * @param ModuleConfig $moduleConfig * @param StreamInterfaceFactory $streamFactory + * @param CacheInterface $cache + * @param SerializerInterface $serializer * @param bool $useModuleConfig * @return void */ @@ -117,6 +128,8 @@ public function __construct( ModuleConfig $moduleConfig, StreamInterfaceFactory $streamFactory, StoreManagerInterface $storeManager, + CacheInterface $cache, + SerializerInterface $serializer, bool $useModuleConfig = false ) { $this->enabled = $fileStorage->checkBucketUsage(); @@ -128,6 +141,8 @@ public function __construct( $this->streamFactory = $streamFactory; $this->storeManager = $storeManager; $this->useModuleConfig = $useModuleConfig; + $this->cache = $cache; + $this->serializer = $serializer; $this->initialize(); } @@ -269,6 +284,13 @@ public function getClient(): StorageClient */ public function getObject(string $path): ?StorageObject { + $cache = $this->cache->load(GcsCache::TYPE_IDENTIFIER); + $cacheGcs = $cache ? $this->serializer->unserialize($cache) : []; + + if (in_array($path, $cacheGcs)) { + return null; + } + if ($this->hasPrefix()) { $prefixedPath = implode(DIRECTORY_SEPARATOR, [ $this->getPrefix(), @@ -311,6 +333,14 @@ public function getObject(string $path): ?StorageObject curl_close($ch); } + //Store path in GcsCache + $this->cache->save( + $this->serializer->serialize(array_merge($cacheGcs, [$path])), + GcsCache::TYPE_IDENTIFIER, + [GcsCache::CACHE_TAG], + 86400 + ); + return $object; } diff --git a/Model/Cache/Type/GcsCache.php b/Model/Cache/Type/GcsCache.php new file mode 100644 index 0000000..829967e --- /dev/null +++ b/Model/Cache/Type/GcsCache.php @@ -0,0 +1,24 @@ +get(self::TYPE_IDENTIFIER), + self::CACHE_TAG + ); + } +} diff --git a/etc/cache.xml b/etc/cache.xml new file mode 100644 index 0000000..40df50a --- /dev/null +++ b/etc/cache.xml @@ -0,0 +1,7 @@ + + + + + Cache for saving GCS path + + From 8c348b648f731c25dd27c0c9895659725f1b821a Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 19 Jul 2023 14:39:51 +0000 Subject: [PATCH 40/62] Correctly handle null responses --- Api/StorageTypeMetadataInterface.php | 2 -- Model/Adapter/StorageObjectManagement.php | 4 ++-- Plugin/Catalog/Product/Gallery/ObjectUploader.php | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Api/StorageTypeMetadataInterface.php b/Api/StorageTypeMetadataInterface.php index 5956964..adebf6e 100644 --- a/Api/StorageTypeMetadataInterface.php +++ b/Api/StorageTypeMetadataInterface.php @@ -18,8 +18,6 @@ namespace AuroraExtensions\GoogleCloudStorage\Api; -use Google\Cloud\Storage\StorageClient; - interface StorageTypeMetadataInterface { /** @constant int STORAGE_MEDIA_GCS */ diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index ac5f2b0..1df688a 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -450,7 +450,7 @@ public function copyObject(string $source, string $target): ?StorageObject /** @var StorageObject $object */ $object = $this->getObject($source); - return $object->exists() ? $object->copy($target) : null; + return ($object !== null && $object->exists()) ? $object->copy($target) : null; } /** @@ -471,7 +471,7 @@ public function renameObject(string $source, string $target): ?StorageObject /** @var StorageObject $object */ $object = $this->getObject($source); - return $object->exists() ? $object->rename($target) : null; + return ($object !== null && $object->exists()) ? $object->rename($target) : null; } /** diff --git a/Plugin/Catalog/Product/Gallery/ObjectUploader.php b/Plugin/Catalog/Product/Gallery/ObjectUploader.php index e887b4e..304d9d1 100644 --- a/Plugin/Catalog/Product/Gallery/ObjectUploader.php +++ b/Plugin/Catalog/Product/Gallery/ObjectUploader.php @@ -91,12 +91,12 @@ public function __construct( /** * @param ExtensionInterface $subject * @param ProductInterface $result - * @return void + * @return ProductInterface */ public function afterExecute( ExtensionInterface $subject, ProductInterface $result - ) { + ): ProductInterface { if (!$this->storageAdapter->isEnabled()) { return $result; } From cb3427f376e876f44796c5f54b752b23046a7155 Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 19 Jul 2023 17:58:27 +0000 Subject: [PATCH 41/62] Fix for getObject on null --- Model/File/Storage/Bucket.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Model/File/Storage/Bucket.php b/Model/File/Storage/Bucket.php index 7228614..52b20e2 100644 --- a/Model/File/Storage/Bucket.php +++ b/Model/File/Storage/Bucket.php @@ -155,10 +155,10 @@ public function setObjects(?ObjectIterator $objects) */ public function loadByFilename(string $relativePath) { - if ($this->getStorage()->objectExists($relativePath)) { + if ($this->getStorage()->objectExists($relativePath) && $object = $this->getStorage()->getObject($relativePath)) { $this->setData('id', $relativePath); $this->setData('filename', $relativePath); - $this->setData('content', $this->getStorage()->getObject($relativePath)->downloadAsString()); + $this->setData('content', $object->downloadAsString()); } else { $this->unsetData(); } From ed08f03221c83e220c4f00ae1950b095b2d4e4ce Mon Sep 17 00:00:00 2001 From: David Windell Date: Thu, 18 Apr 2024 09:11:57 +0000 Subject: [PATCH 42/62] Change PSR version dependencies --- composer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index cddb008..7efbdbb 100644 --- a/composer.json +++ b/composer.json @@ -4,8 +4,8 @@ "type": "magento2-module", "license": "MIT", "require": { - "psr/http-message": "~1.0", - "psr/log": "~1.0", + "psr/http-message": ">=1.0", + "psr/log": ">=1.0", "google/cloud-storage": "~1.27.0", "magento/framework": "^103", "magento/module-media-storage": "^100", From e1ac0814c18e9aefadfc126f16db45f3b498029e Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 1 May 2024 16:20:44 +0100 Subject: [PATCH 43/62] Update image-catcher.phtml (#3) Fix event listener breaking on hyva or breeze magento 2. Co-authored-by: Beniamin Sinca --- view/frontend/templates/image-catcher.phtml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/view/frontend/templates/image-catcher.phtml b/view/frontend/templates/image-catcher.phtml index 7f91daf..10001bc 100644 --- a/view/frontend/templates/image-catcher.phtml +++ b/view/frontend/templates/image-catcher.phtml @@ -12,7 +12,7 @@ replacedImages.push(url); event.target.src = url.replace(cdnUrl, mediaUrl) + '?imgstore=getStoreCode() ?>'; - if(event.target.offsetParent?.tagName.toLowerCase() == 'picture') { + if(event.target.offsetParent?.tagName?.toLowerCase() == 'picture') { const sources = event.target.offsetParent.querySelectorAll('source'); sources.forEach(function(source) { let url = source.srcset.split('?')[0]; @@ -29,8 +29,9 @@ window.addEventListener('load', function () { // Only run the mage/gallery require on product page - if (document.body.classList.contains('catalog-product-view')) { + if (document.body.classList.contains('catalog-product-view') && window.require) { require(['jquery', 'mage/gallery/gallery'], function($, gallery){ + if (!$('[data-gallery-role=gallery-placeholder]')) return; $('[data-gallery-role=gallery-placeholder]').on('gallery:loaded', async function () { const api = $(this).data('gallery'); From 25bfe81bb8dc98d40567800cfbb6ba4c6a845628 Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 22 May 2024 12:02:15 +0100 Subject: [PATCH 44/62] Update cache.xml --- etc/cache.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/cache.xml b/etc/cache.xml index 40df50a..91da8fb 100644 --- a/etc/cache.xml +++ b/etc/cache.xml @@ -1,7 +1,7 @@ - + Cache for saving GCS path From 3ac558be35955bdab716dbe8c8ce1feb51e835fc Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 22 May 2024 13:23:13 +0100 Subject: [PATCH 45/62] Remove TTL on GCS cache --- Model/Adapter/StorageObjectManagement.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 1df688a..982c920 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -337,8 +337,7 @@ public function getObject(string $path): ?StorageObject $this->cache->save( $this->serializer->serialize(array_merge($cacheGcs, [$path])), GcsCache::TYPE_IDENTIFIER, - [GcsCache::CACHE_TAG], - 86400 + [GcsCache::CACHE_TAG] ); return $object; From 46357a64e9a926137df39dee21fb8fa5fb97b30c Mon Sep 17 00:00:00 2001 From: David Windell Date: Thu, 4 Jul 2024 12:15:05 +0100 Subject: [PATCH 46/62] Prevent JS errors when no tagName --- view/frontend/templates/image-catcher.phtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/view/frontend/templates/image-catcher.phtml b/view/frontend/templates/image-catcher.phtml index 10001bc..e488ff8 100644 --- a/view/frontend/templates/image-catcher.phtml +++ b/view/frontend/templates/image-catcher.phtml @@ -6,7 +6,7 @@ var replacedImages = []; window.addEventListener('error', (event) => { - if (event.target.tagName.toLowerCase() !== 'img') return; + if (event.target.tagName?.toLowerCase() !== 'img') return; const url = event.target.src.split('?')[0]; if (replacedImages.includes(url)) return; replacedImages.push(url); From d82e238451fb15c5d51caeeb48f4c264f0a5081e Mon Sep 17 00:00:00 2001 From: Bruno Dias Date: Thu, 12 Sep 2024 16:33:17 +0100 Subject: [PATCH 47/62] Image fallback queue --- Model/Consumer/ImageConsumer.php | 16 ++++++++++++++++ Model/Consumer/ImagePublisher.php | 30 ++++++++++++++++++++++++++++++ etc/communication.xml | 4 ++++ etc/crontab.xml | 8 ++++++++ etc/queue_consumer.xml | 4 ++++ etc/queue_publisher.xml | 6 ++++++ etc/queue_topology.xml | 6 ++++++ 7 files changed, 74 insertions(+) create mode 100644 Model/Consumer/ImageConsumer.php create mode 100644 Model/Consumer/ImagePublisher.php create mode 100644 etc/communication.xml create mode 100644 etc/crontab.xml create mode 100644 etc/queue_consumer.xml create mode 100644 etc/queue_publisher.xml create mode 100644 etc/queue_topology.xml diff --git a/Model/Consumer/ImageConsumer.php b/Model/Consumer/ImageConsumer.php new file mode 100644 index 0000000..441ea75 --- /dev/null +++ b/Model/Consumer/ImageConsumer.php @@ -0,0 +1,16 @@ +publisher = $publisher; + } + + + public function execute(OrderInterface $order) + { + $this->publisher->publish(self::QUEUE_NAME, $order); + } +} \ No newline at end of file diff --git a/etc/communication.xml b/etc/communication.xml new file mode 100644 index 0000000..be89bbf --- /dev/null +++ b/etc/communication.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/etc/crontab.xml b/etc/crontab.xml new file mode 100644 index 0000000..175f342 --- /dev/null +++ b/etc/crontab.xml @@ -0,0 +1,8 @@ + + + + + * * * * * + + + \ No newline at end of file diff --git a/etc/queue_consumer.xml b/etc/queue_consumer.xml new file mode 100644 index 0000000..5ad29fb --- /dev/null +++ b/etc/queue_consumer.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/etc/queue_publisher.xml b/etc/queue_publisher.xml new file mode 100644 index 0000000..e746c98 --- /dev/null +++ b/etc/queue_publisher.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/etc/queue_topology.xml b/etc/queue_topology.xml new file mode 100644 index 0000000..4b4e23b --- /dev/null +++ b/etc/queue_topology.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file From 42f65f5136ac36ef0e4a2ea338424579844e60e3 Mon Sep 17 00:00:00 2001 From: Bruno Dias Date: Thu, 12 Sep 2024 16:38:26 +0100 Subject: [PATCH 48/62] Image fallback queue --- Model/Consumer/ImageConsumer.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Model/Consumer/ImageConsumer.php b/Model/Consumer/ImageConsumer.php index 441ea75..44ddbb4 100644 --- a/Model/Consumer/ImageConsumer.php +++ b/Model/Consumer/ImageConsumer.php @@ -2,13 +2,15 @@ namespace AuroraExtensions\GoogleCloudStorage\Model\Consumer; +use AuroraExtensions\GoogleCloudStorage\Api\StorageObjectManagementInterface; + class ImageConsumer { /** - * @param OrderInterface $order + * @param StorageObjectManagementInterface $storage */ - public function processMessage(OrderInterface $order): void + public function processMessage(StorageObjectManagementInterface $storage): void { // Implement your logic to here // This method will be executed when a message is available in the queue From c3087876288c8d4bded57a10c8876f4211d6a410 Mon Sep 17 00:00:00 2001 From: Bruno Dias Date: Thu, 12 Sep 2024 16:44:37 +0100 Subject: [PATCH 49/62] Image fallback queue --- etc/queue_topology.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/queue_topology.xml b/etc/queue_topology.xml index 4b4e23b..8d1f5bd 100644 --- a/etc/queue_topology.xml +++ b/etc/queue_topology.xml @@ -1,6 +1,6 @@ - + \ No newline at end of file From 1f946c9d5b88c36c81347937fffac2af217dda92 Mon Sep 17 00:00:00 2001 From: Bruno Dias Date: Fri, 13 Sep 2024 15:41:31 +0100 Subject: [PATCH 50/62] Improve queue code --- Model/Adapter/StorageObjectManagement.php | 16 +++--------- Model/Consumer/ImageConsumer.php | 31 ++++++++++++++++------- Model/Consumer/ImagePublisher.php | 24 +++++------------- etc/crontab.xml | 8 ------ etc/queue_topology.xml | 2 +- 5 files changed, 33 insertions(+), 48 deletions(-) delete mode 100644 etc/crontab.xml diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 982c920..fa7641c 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -18,6 +18,7 @@ namespace AuroraExtensions\GoogleCloudStorage\Model\Adapter; +use AuroraExtensions\GoogleCloudStorage\Model\Consumer\ImagePublisher; use AuroraExtensions\GoogleCloudStorage\{ Api\StorageObjectManagementInterface, Api\StorageObjectPathResolverInterface, @@ -130,6 +131,7 @@ public function __construct( StoreManagerInterface $storeManager, CacheInterface $cache, SerializerInterface $serializer, + protected ImagePublisher $imageQueuePublisher, bool $useModuleConfig = false ) { $this->enabled = $fileStorage->checkBucketUsage(); @@ -317,20 +319,8 @@ public function getObject(string $path): ?StorageObject $fallback = $fallback['default']; } } - /* Attempt to load the image from fallback URL and upload to GCS */ - $ch = curl_init($fallback . $path); - curl_setopt($ch, CURLOPT_TIMEOUT, 5); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - $content = curl_exec($ch); - - if ($content && curl_getinfo($ch, CURLINFO_HTTP_CODE) === 200) { - $object = $this->uploadObject($content, [ - 'name' => $prefixedPath, - 'predefinedAcl' => $this->getObjectAclPolicy() - ]); - } - curl_close($ch); + $this->imageQueuePublisher->execute(['url' => $fallback . $path, 'prefixedPath' => $prefixedPath]); } //Store path in GcsCache diff --git a/Model/Consumer/ImageConsumer.php b/Model/Consumer/ImageConsumer.php index 44ddbb4..52f204b 100644 --- a/Model/Consumer/ImageConsumer.php +++ b/Model/Consumer/ImageConsumer.php @@ -2,17 +2,30 @@ namespace AuroraExtensions\GoogleCloudStorage\Model\Consumer; -use AuroraExtensions\GoogleCloudStorage\Api\StorageObjectManagementInterface; +use AuroraExtensions\GoogleCloudStorage\Model\Adapter\StorageObjectManagement; class ImageConsumer { - - /** - * @param StorageObjectManagementInterface $storage - */ - public function processMessage(StorageObjectManagementInterface $storage): void + public function __construct( + protected StorageObjectManagement $storageObjectManagement + ) { + } + + public function processMessage($data): void { - // Implement your logic to here - // This method will be executed when a message is available in the queue + /* Attempt to load the image from fallback URL and upload to GCS */ + $ch = curl_init($data['url']); + curl_setopt($ch, CURLOPT_TIMEOUT, 5); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $content = curl_exec($ch); + + if ($content && curl_getinfo($ch, CURLINFO_HTTP_CODE) === 200) { + $object = $this->storageObjectManagement->uploadObject($content, [ + 'name' => $data['prefixedPath'], + 'predefinedAcl' => $this->storageObjectManagement->getObjectAclPolicy() + ]); + } + + curl_close($ch); } -} \ No newline at end of file +} diff --git a/Model/Consumer/ImagePublisher.php b/Model/Consumer/ImagePublisher.php index f388b2e..dca55fb 100644 --- a/Model/Consumer/ImagePublisher.php +++ b/Model/Consumer/ImagePublisher.php @@ -2,29 +2,19 @@ namespace AuroraExtensions\GoogleCloudStorage\Model\Consumer; -use Magento\Framework\MessageQueue\PublisherInterface ; +use Magento\Framework\MessageQueue\PublisherInterface; class ImagePublisher { - const QUEUE_NAME = 'outeredge.fallback.image.queue'; - /** - * @var \Magento\Framework\MessageQueue\PublisherInterface - */ - private $publisher; - - /** - * @param \Magento\Framework\MessageQueue\PublisherInterface $publisher - */ - public function __construct(PublisherInterface $publisher) - { - $this->publisher = $publisher; + public function __construct( + protected PublisherInterface $publisher + ) { } - - public function execute(OrderInterface $order) + public function execute($data) { - $this->publisher->publish(self::QUEUE_NAME, $order); + $this->publisher->publish(self::QUEUE_NAME, $data); } -} \ No newline at end of file +} diff --git a/etc/crontab.xml b/etc/crontab.xml deleted file mode 100644 index 175f342..0000000 --- a/etc/crontab.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - * * * * * - - - \ No newline at end of file diff --git a/etc/queue_topology.xml b/etc/queue_topology.xml index 8d1f5bd..704ce7a 100644 --- a/etc/queue_topology.xml +++ b/etc/queue_topology.xml @@ -3,4 +3,4 @@ - \ No newline at end of file + From 488637309168860cc77db7e78135752c10688beb Mon Sep 17 00:00:00 2001 From: Bruno Dias Date: Wed, 18 Sep 2024 11:25:30 +0100 Subject: [PATCH 51/62] Fix data type of communication --- Model/Consumer/ImageConsumer.php | 4 ++-- etc/communication.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Model/Consumer/ImageConsumer.php b/Model/Consumer/ImageConsumer.php index 52f204b..990a945 100644 --- a/Model/Consumer/ImageConsumer.php +++ b/Model/Consumer/ImageConsumer.php @@ -14,14 +14,14 @@ public function __construct( public function processMessage($data): void { /* Attempt to load the image from fallback URL and upload to GCS */ - $ch = curl_init($data['url']); + $ch = curl_init($data[0]); curl_setopt($ch, CURLOPT_TIMEOUT, 5); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); $content = curl_exec($ch); if ($content && curl_getinfo($ch, CURLINFO_HTTP_CODE) === 200) { $object = $this->storageObjectManagement->uploadObject($content, [ - 'name' => $data['prefixedPath'], + 'name' => $data[1], 'predefinedAcl' => $this->storageObjectManagement->getObjectAclPolicy() ]); } diff --git a/etc/communication.xml b/etc/communication.xml index be89bbf..9a212e6 100644 --- a/etc/communication.xml +++ b/etc/communication.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file From cdd0c78ffd813b67fa42fdc3791a07688eaa9d74 Mon Sep 17 00:00:00 2001 From: David Windell Date: Fri, 20 Sep 2024 15:42:34 +0000 Subject: [PATCH 52/62] Add User Agent and tweak timeouts --- Model/Consumer/ImageConsumer.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Model/Consumer/ImageConsumer.php b/Model/Consumer/ImageConsumer.php index 990a945..ae3c89a 100644 --- a/Model/Consumer/ImageConsumer.php +++ b/Model/Consumer/ImageConsumer.php @@ -6,6 +6,8 @@ class ImageConsumer { + const USER_AGENT = 'outeredge/gcs'; + public function __construct( protected StorageObjectManagement $storageObjectManagement ) { @@ -15,12 +17,14 @@ public function processMessage($data): void { /* Attempt to load the image from fallback URL and upload to GCS */ $ch = curl_init($data[0]); - curl_setopt($ch, CURLOPT_TIMEOUT, 5); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + curl_setopt($ch, CURLOPT_TIMEOUT, 30); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_USERAGENT, self::USER_AGENT); $content = curl_exec($ch); if ($content && curl_getinfo($ch, CURLINFO_HTTP_CODE) === 200) { - $object = $this->storageObjectManagement->uploadObject($content, [ + $this->storageObjectManagement->uploadObject($content, [ 'name' => $data[1], 'predefinedAcl' => $this->storageObjectManagement->getObjectAclPolicy() ]); From c7c8a3ac79554bb649d8c665bec61bef84d54330 Mon Sep 17 00:00:00 2001 From: David Windell Date: Fri, 20 Sep 2024 15:55:50 +0000 Subject: [PATCH 53/62] Minor performance improvement --- Model/Adapter/StorageObjectManagement.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index fa7641c..c39632f 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -303,7 +303,7 @@ public function getObject(string $path): ?StorageObject $object = $this->bucket->object($prefixedPath); $fallback = $this->deploymentConfig->get('storage/fallback_url'); - if (!$object->exists() && $fallback) { + if ($fallback && !$object->exists()) { if (is_array($fallback)) { $storecode = $this->storeManager->getStore()->getCode(); From a538122059ecfefb08a61f25888d8e6949a162ad Mon Sep 17 00:00:00 2001 From: David Windell Date: Fri, 20 Sep 2024 16:08:44 +0000 Subject: [PATCH 54/62] Cache earlier to avoid repeat requests --- Model/Adapter/StorageObjectManagement.php | 27 +++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index c39632f..ff889c9 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -293,6 +293,12 @@ public function getObject(string $path): ?StorageObject return null; } + $this->cache->save( + $this->serializer->serialize(array_merge($cacheGcs, [$path])), + GcsCache::TYPE_IDENTIFIER, + [GcsCache::CACHE_TAG] + ); + if ($this->hasPrefix()) { $prefixedPath = implode(DIRECTORY_SEPARATOR, [ $this->getPrefix(), @@ -323,13 +329,6 @@ public function getObject(string $path): ?StorageObject $this->imageQueuePublisher->execute(['url' => $fallback . $path, 'prefixedPath' => $prefixedPath]); } - //Store path in GcsCache - $this->cache->save( - $this->serializer->serialize(array_merge($cacheGcs, [$path])), - GcsCache::TYPE_IDENTIFIER, - [GcsCache::CACHE_TAG] - ); - return $object; } @@ -365,6 +364,20 @@ public function objectExists(string $path): bool return false; } + // Don't waste requests if an attempt has already been made + $cache = $this->cache->load(GcsCache::TYPE_IDENTIFIER); + $cacheGcs = $cache ? $this->serializer->unserialize($cache) : []; + + if (in_array($path, $cacheGcs)) { + return false; + } + + $this->cache->save( + $this->serializer->serialize(array_merge($cacheGcs, [$path])), + GcsCache::TYPE_IDENTIFIER, + [GcsCache::CACHE_TAG] + ); + /** @var StorageObject|null $object */ $object = $this->getObject($path); return ($object !== null && $object->exists()); From 910c7ba73f8e30a6af91322fb52844e50fa29f75 Mon Sep 17 00:00:00 2001 From: David Windell Date: Fri, 20 Sep 2024 16:49:33 +0000 Subject: [PATCH 55/62] More optimisation --- Model/Adapter/StorageObjectManagement.php | 25 ++++++++++------------- Model/File/Storage/Bucket.php | 2 +- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index ff889c9..9427048 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -293,12 +293,6 @@ public function getObject(string $path): ?StorageObject return null; } - $this->cache->save( - $this->serializer->serialize(array_merge($cacheGcs, [$path])), - GcsCache::TYPE_IDENTIFIER, - [GcsCache::CACHE_TAG] - ); - if ($this->hasPrefix()) { $prefixedPath = implode(DIRECTORY_SEPARATOR, [ $this->getPrefix(), @@ -308,8 +302,11 @@ public function getObject(string $path): ?StorageObject $object = $this->bucket->object($prefixedPath); $fallback = $this->deploymentConfig->get('storage/fallback_url'); + $exists = false; - if ($fallback && !$object->exists()) { + if ($object->exists()) { + $exists = true; + } elseif ($fallback) { if (is_array($fallback)) { $storecode = $this->storeManager->getStore()->getCode(); @@ -329,6 +326,12 @@ public function getObject(string $path): ?StorageObject $this->imageQueuePublisher->execute(['url' => $fallback . $path, 'prefixedPath' => $prefixedPath]); } + $this->cache->save( + $this->serializer->serialize(array_merge($cacheGcs, [$path => $exists])), + GcsCache::TYPE_IDENTIFIER, + [GcsCache::CACHE_TAG] + ); + return $object; } @@ -369,15 +372,9 @@ public function objectExists(string $path): bool $cacheGcs = $cache ? $this->serializer->unserialize($cache) : []; if (in_array($path, $cacheGcs)) { - return false; + return $cacheGcs[$path]; } - $this->cache->save( - $this->serializer->serialize(array_merge($cacheGcs, [$path])), - GcsCache::TYPE_IDENTIFIER, - [GcsCache::CACHE_TAG] - ); - /** @var StorageObject|null $object */ $object = $this->getObject($path); return ($object !== null && $object->exists()); diff --git a/Model/File/Storage/Bucket.php b/Model/File/Storage/Bucket.php index 52b20e2..638dbfe 100644 --- a/Model/File/Storage/Bucket.php +++ b/Model/File/Storage/Bucket.php @@ -155,7 +155,7 @@ public function setObjects(?ObjectIterator $objects) */ public function loadByFilename(string $relativePath) { - if ($this->getStorage()->objectExists($relativePath) && $object = $this->getStorage()->getObject($relativePath)) { + if ($object = $this->getStorage()->getObject($relativePath)) { $this->setData('id', $relativePath); $this->setData('filename', $relativePath); $this->setData('content', $object->downloadAsString()); From cc248ef4dae2aecfd54f042b26406b899ed3b683 Mon Sep 17 00:00:00 2001 From: David Windell Date: Fri, 20 Sep 2024 16:58:09 +0000 Subject: [PATCH 56/62] Doh, abandon consumer because we don't run them in dev --- Model/Adapter/StorageObjectManagement.php | 19 ++++++++++-- Model/Consumer/ImageConsumer.php | 35 ----------------------- Model/Consumer/ImagePublisher.php | 20 ------------- etc/communication.xml | 4 --- etc/queue_consumer.xml | 4 --- etc/queue_publisher.xml | 6 ---- etc/queue_topology.xml | 6 ---- 7 files changed, 16 insertions(+), 78 deletions(-) delete mode 100644 Model/Consumer/ImageConsumer.php delete mode 100644 Model/Consumer/ImagePublisher.php delete mode 100644 etc/communication.xml delete mode 100644 etc/queue_consumer.xml delete mode 100644 etc/queue_publisher.xml delete mode 100644 etc/queue_topology.xml diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 9427048..8abe2fe 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -18,7 +18,6 @@ namespace AuroraExtensions\GoogleCloudStorage\Model\Adapter; -use AuroraExtensions\GoogleCloudStorage\Model\Consumer\ImagePublisher; use AuroraExtensions\GoogleCloudStorage\{ Api\StorageObjectManagementInterface, Api\StorageObjectPathResolverInterface, @@ -131,7 +130,6 @@ public function __construct( StoreManagerInterface $storeManager, CacheInterface $cache, SerializerInterface $serializer, - protected ImagePublisher $imageQueuePublisher, bool $useModuleConfig = false ) { $this->enabled = $fileStorage->checkBucketUsage(); @@ -323,7 +321,22 @@ public function getObject(string $path): ?StorageObject } } - $this->imageQueuePublisher->execute(['url' => $fallback . $path, 'prefixedPath' => $prefixedPath]); + /* Attempt to load the image from fallback URL and upload to GCS */ + + // @todo move this to a shell background process + $ch = curl_init($data[0]); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + curl_setopt($ch, CURLOPT_TIMEOUT, 30); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_USERAGENT, self::USER_AGENT); + $content = curl_exec($ch); + if ($content && curl_getinfo($ch, CURLINFO_HTTP_CODE) === 200) { + $this->storageObjectManagement->uploadObject($content, [ + 'name' => $data[1], + 'predefinedAcl' => $this->storageObjectManagement->getObjectAclPolicy() + ]); + } + curl_close($ch); } $this->cache->save( diff --git a/Model/Consumer/ImageConsumer.php b/Model/Consumer/ImageConsumer.php deleted file mode 100644 index ae3c89a..0000000 --- a/Model/Consumer/ImageConsumer.php +++ /dev/null @@ -1,35 +0,0 @@ -storageObjectManagement->uploadObject($content, [ - 'name' => $data[1], - 'predefinedAcl' => $this->storageObjectManagement->getObjectAclPolicy() - ]); - } - - curl_close($ch); - } -} diff --git a/Model/Consumer/ImagePublisher.php b/Model/Consumer/ImagePublisher.php deleted file mode 100644 index dca55fb..0000000 --- a/Model/Consumer/ImagePublisher.php +++ /dev/null @@ -1,20 +0,0 @@ -publisher->publish(self::QUEUE_NAME, $data); - } -} diff --git a/etc/communication.xml b/etc/communication.xml deleted file mode 100644 index 9a212e6..0000000 --- a/etc/communication.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/etc/queue_consumer.xml b/etc/queue_consumer.xml deleted file mode 100644 index 5ad29fb..0000000 --- a/etc/queue_consumer.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/etc/queue_publisher.xml b/etc/queue_publisher.xml deleted file mode 100644 index e746c98..0000000 --- a/etc/queue_publisher.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/etc/queue_topology.xml b/etc/queue_topology.xml deleted file mode 100644 index 704ce7a..0000000 --- a/etc/queue_topology.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - From ebacb66f3a988f7b01e08ffbd73185ed11fffdca Mon Sep 17 00:00:00 2001 From: David Windell Date: Fri, 20 Sep 2024 17:09:21 +0000 Subject: [PATCH 57/62] Add user agent --- Model/Adapter/StorageObjectManagement.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 8abe2fe..bbac8a0 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -62,6 +62,8 @@ class StorageObjectManagement implements StorageObjectManagementInterface, StorageObjectPathResolverInterface { + const USER_AGENT = 'outeredge/gcs'; + /** * @var ModuleConfig $moduleConfig * @method ModuleConfig getConfig() @@ -322,18 +324,17 @@ public function getObject(string $path): ?StorageObject } /* Attempt to load the image from fallback URL and upload to GCS */ - // @todo move this to a shell background process - $ch = curl_init($data[0]); + $ch = curl_init($fallback . $path); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); curl_setopt($ch, CURLOPT_TIMEOUT, 30); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_USERAGENT, self::USER_AGENT); $content = curl_exec($ch); if ($content && curl_getinfo($ch, CURLINFO_HTTP_CODE) === 200) { - $this->storageObjectManagement->uploadObject($content, [ - 'name' => $data[1], - 'predefinedAcl' => $this->storageObjectManagement->getObjectAclPolicy() + $this->uploadObject($content, [ + 'name' => $prefixedPath, + 'predefinedAcl' => $this->getObjectAclPolicy() ]); } curl_close($ch); From 6a94185c52ad8cdd8756c03912c0bdf7a49d7c57 Mon Sep 17 00:00:00 2001 From: Bruno Dias Date: Mon, 23 Sep 2024 11:41:44 +0100 Subject: [PATCH 58/62] Added Command line download --- Console/Command/DownloadImage.php | 75 +++++++++++++++++++++++ Model/Adapter/StorageObjectManagement.php | 15 +---- etc/di.xml | 7 +++ 3 files changed, 84 insertions(+), 13 deletions(-) create mode 100644 Console/Command/DownloadImage.php diff --git a/Console/Command/DownloadImage.php b/Console/Command/DownloadImage.php new file mode 100644 index 0000000..b610e54 --- /dev/null +++ b/Console/Command/DownloadImage.php @@ -0,0 +1,75 @@ +setName('outeredge:gcs:download'); + $this->setDescription('Download Image background process'); + + parent::configure(); + } + + /** + * Execute the command + * + * @param InputInterface $input + * @param OutputInterface $output + * + * @return int + */ + protected function execute(InputInterface $input, OutputInterface $output): int + { + $exitCode = 0; + + $url = $input->getOption(self::URL); + $prefixedPath = $input->getOption(self::PREFIXEDPATH); + + try { + $ch = curl_init($ulr); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + curl_setopt($ch, CURLOPT_TIMEOUT, 30); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_USERAGENT, self::USER_AGENT); + $content = curl_exec($ch); + if ($content && curl_getinfo($ch, CURLINFO_HTTP_CODE) === 200) { + $this->storageObjectManagement->uploadObject($content, [ + 'name' => $prefixedPath, + 'predefinedAcl' => $this->storageObjectManagement->getObjectAclPolicy() + ]); + } + curl_close($ch); + + $output->writeln('Successfully donwload image'); + } catch (LocalizedException $e) { + $output->writeln(sprintf( + '%s', + $e->getMessage() + )); + $exitCode = 1; + } + + return $exitCode; + } +} diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index bbac8a0..0bc7747 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -325,19 +325,8 @@ public function getObject(string $path): ?StorageObject /* Attempt to load the image from fallback URL and upload to GCS */ // @todo move this to a shell background process - $ch = curl_init($fallback . $path); - curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); - curl_setopt($ch, CURLOPT_TIMEOUT, 30); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_USERAGENT, self::USER_AGENT); - $content = curl_exec($ch); - if ($content && curl_getinfo($ch, CURLINFO_HTTP_CODE) === 200) { - $this->uploadObject($content, [ - 'name' => $prefixedPath, - 'predefinedAcl' => $this->getObjectAclPolicy() - ]); - } - curl_close($ch); + $url = $fallback . $path; + exec('bash -c "exec nohup setsid bin/magento outeredge:gcs:download --url=$url --prefixedPath=$prefixedPath > /dev/null 2>&1 &"'); } $this->cache->save( diff --git a/etc/di.xml b/etc/di.xml index d3241d4..071c7a7 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -126,4 +126,11 @@ type="AuroraExtensions\GoogleCloudStorage\Plugin\Catalog\Category" sortOrder="10"/> + + + + AuroraExtensions\GoogleCloudStorage\Console\Command\DownloadImage + + + From e02d56499388ae601748b38fc544736e624d77b0 Mon Sep 17 00:00:00 2001 From: Bruno Dias Date: Mon, 23 Sep 2024 13:10:51 +0100 Subject: [PATCH 59/62] Added Command line download --- Console/Command/DownloadImage.php | 9 ++++++--- Model/Adapter/StorageObjectManagement.php | 7 +++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Console/Command/DownloadImage.php b/Console/Command/DownloadImage.php index b610e54..4b572f1 100644 --- a/Console/Command/DownloadImage.php +++ b/Console/Command/DownloadImage.php @@ -9,6 +9,7 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use AuroraExtensions\GoogleCloudStorage\Model\Adapter\StorageObjectManagement; +use Symfony\Component\Console\Input\InputArgument; class DownloadImage extends Command { @@ -27,6 +28,8 @@ protected function configure(): void { $this->setName('outeredge:gcs:download'); $this->setDescription('Download Image background process'); + $this->addArgument('url', InputArgument::REQUIRED); + $this->addArgument('prefixedPath', InputArgument::REQUIRED); parent::configure(); } @@ -43,11 +46,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int { $exitCode = 0; - $url = $input->getOption(self::URL); - $prefixedPath = $input->getOption(self::PREFIXEDPATH); + $url = $input->getArgument(self::URL); + $prefixedPath = $input->getArgument(self::PREFIXEDPATH); try { - $ch = curl_init($ulr); + $ch = curl_init($url); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); curl_setopt($ch, CURLOPT_TIMEOUT, 30); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 0bc7747..83d686f 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -323,10 +323,9 @@ public function getObject(string $path): ?StorageObject } } - /* Attempt to load the image from fallback URL and upload to GCS */ - // @todo move this to a shell background process - $url = $fallback . $path; - exec('bash -c "exec nohup setsid bin/magento outeredge:gcs:download --url=$url --prefixedPath=$prefixedPath > /dev/null 2>&1 &"'); + /* Load the image in a shell background process */ + $url = $fallback . $path; + exec("bash -c 'exec nohup setsid bin/magento outeredge:gcs:download $url $prefixedPath > /dev/null 2>&1 &'"); } $this->cache->save( From cc978ab9a99012acc2c447d2910282d82388c20b Mon Sep 17 00:00:00 2001 From: Bruno Dias Date: Mon, 23 Sep 2024 15:59:13 +0100 Subject: [PATCH 60/62] Added Command line download --- Console/Command/DownloadImage.php | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Console/Command/DownloadImage.php b/Console/Command/DownloadImage.php index 4b572f1..76b6857 100644 --- a/Console/Command/DownloadImage.php +++ b/Console/Command/DownloadImage.php @@ -10,6 +10,7 @@ use Symfony\Component\Console\Output\OutputInterface; use AuroraExtensions\GoogleCloudStorage\Model\Adapter\StorageObjectManagement; use Symfony\Component\Console\Input\InputArgument; +use Psr\Log\LoggerInterface; class DownloadImage extends Command { @@ -19,6 +20,7 @@ class DownloadImage extends Command public function __construct( protected StorageObjectManagement $storageObjectManagement, + protected LoggerInterface $logger, string $name = null ) { parent::__construct($name); @@ -63,13 +65,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int ]); } curl_close($ch); - - $output->writeln('Successfully donwload image'); } catch (LocalizedException $e) { - $output->writeln(sprintf( - '%s', - $e->getMessage() - )); + $this->logger->critical($e->getMessage()); $exitCode = 1; } From c9a501da64eaf706089097d82e564d7a494fb820 Mon Sep 17 00:00:00 2001 From: Bruno Dias Date: Mon, 23 Sep 2024 15:59:23 +0100 Subject: [PATCH 61/62] Added Command line download --- Console/Command/DownloadImage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Console/Command/DownloadImage.php b/Console/Command/DownloadImage.php index 76b6857..c2e1aae 100644 --- a/Console/Command/DownloadImage.php +++ b/Console/Command/DownloadImage.php @@ -66,7 +66,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int } curl_close($ch); } catch (LocalizedException $e) { - $this->logger->critical($e->getMessage()); + $this->logger->error($e->getMessage()); $exitCode = 1; } From 8da39194af5896da367c889492d39de8c8bbd3ed Mon Sep 17 00:00:00 2001 From: David Windell Date: Wed, 2 Oct 2024 17:20:59 +0100 Subject: [PATCH 62/62] Improve background process code for multistores --- Model/Adapter/StorageObjectManagement.php | 24 +++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index 83d686f..d044bff 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -307,15 +307,16 @@ public function getObject(string $path): ?StorageObject if ($object->exists()) { $exists = true; } elseif ($fallback) { - if (is_array($fallback)) { - $storecode = $this->storeManager->getStore()->getCode(); + $storecode = $this->storeManager->getStore()->getCode(); - if ($storecode == 'admin' && stristr($_SERVER['REQUEST_URI'], '_admin')) { - $storecode = str_replace('_admin', '', explode('/', ltrim($_SERVER['REQUEST_URI'], '/'))[0]); - } + if ($storecode == 'admin' && stristr($_SERVER['REQUEST_URI'], '_admin')) { + $storecode = str_replace('_admin', '', explode('/', ltrim($_SERVER['REQUEST_URI'], '/'))[0]); + } + if (is_array($fallback)) { if (isset($_GET['imgstore']) && isset($fallback[$_GET['imgstore']])) { - $fallback = $fallback[$_GET['imgstore']]; + $fallback = $fallback[$_GET['imgstore']]; + $storecode = $_GET['imgstore']; } elseif (isset($fallback[$storecode])) { $fallback = $fallback[$storecode]; } else { @@ -324,8 +325,15 @@ public function getObject(string $path): ?StorageObject } /* Load the image in a shell background process */ - $url = $fallback . $path; - exec("bash -c 'exec nohup setsid bin/magento outeredge:gcs:download $url $prefixedPath > /dev/null 2>&1 &'"); + $url = $fallback . $path; + $cmd = sprintf( + 'MAGE_RUN_CODE=%s bin/magento outeredge:gcs:download %s %s', + escapeshellarg($storecode), + escapeshellarg($url), + escapeshellarg($prefixedPath) + ); + + shell_exec(sprintf('%s > /dev/null 2>&1 &', $cmd)); } $this->cache->save(