diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 4c933b8..0000000 --- a/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.sw[po] diff --git a/Api/LocalizedScopeDeploymentConfigInterface.php b/Api/LocalizedScopeDeploymentConfigInterface.php new file mode 100644 index 0000000..f372966 --- /dev/null +++ b/Api/LocalizedScopeDeploymentConfigInterface.php @@ -0,0 +1,28 @@ + + * @license MIT + */ +declare(strict_types=1); + +namespace AuroraExtensions\GoogleCloudStorage\Api; + +interface LocalizedScopeDeploymentConfigInterface +{ + /** + * @param string|null $path + * @return mixed + */ + public function get(?string $path); +} \ No newline at end of file diff --git a/Api/StorageObjectManagementInterface.php b/Api/StorageObjectManagementInterface.php index 4b62bd9..4bda908 100644 --- a/Api/StorageObjectManagementInterface.php +++ b/Api/StorageObjectManagementInterface.php @@ -67,4 +67,9 @@ public function deleteObject(string $path): bool; * @return \AuroraExtensions\GoogleCloudStorage\Api\StorageObjectManagementInterface */ public function deleteAllObjects(array $options): StorageObjectManagementInterface; + + /** + * @return bool + */ + public function isEnabled(): bool; } diff --git a/Api/StorageTypeMetadataInterface.php b/Api/StorageTypeMetadataInterface.php index 5956964..adebf6e 100644 --- a/Api/StorageTypeMetadataInterface.php +++ b/Api/StorageTypeMetadataInterface.php @@ -18,8 +18,6 @@ namespace AuroraExtensions\GoogleCloudStorage\Api; -use Google\Cloud\Storage\StorageClient; - interface StorageTypeMetadataInterface { /** @constant int STORAGE_MEDIA_GCS */ diff --git a/Block/ImageCatcher.php b/Block/ImageCatcher.php new file mode 100644 index 0000000..a8cc51a --- /dev/null +++ b/Block/ImageCatcher.php @@ -0,0 +1,59 @@ +storeManager = $storeManager; + $this->storage = $storage; + } + + public function getMediaUrl() + { + return $this->storeManager->getStore()->getBaseUrl(UrlInterface::URL_TYPE_MEDIA); + } + + public function getBaseMediaUrl() + { + $baseUrl = parse_url($this->storeManager->getStore()->getBaseUrl(UrlInterface::URL_TYPE_WEB)); + return $baseUrl['scheme'] . '://' . $baseUrl['host'] . '/' . DirectoryList::MEDIA . DIRECTORY_SEPARATOR; + } + + public function isEnabled() + { + return $this->storage->checkBucketUsage(); + } + + public function getStoreCode() + { + return $this->storeManager->getStore()->getCode(); + } +} diff --git a/Console/Command/DownloadImage.php b/Console/Command/DownloadImage.php new file mode 100644 index 0000000..c2e1aae --- /dev/null +++ b/Console/Command/DownloadImage.php @@ -0,0 +1,75 @@ +setName('outeredge:gcs:download'); + $this->setDescription('Download Image background process'); + $this->addArgument('url', InputArgument::REQUIRED); + $this->addArgument('prefixedPath', InputArgument::REQUIRED); + + parent::configure(); + } + + /** + * Execute the command + * + * @param InputInterface $input + * @param OutputInterface $output + * + * @return int + */ + protected function execute(InputInterface $input, OutputInterface $output): int + { + $exitCode = 0; + + $url = $input->getArgument(self::URL); + $prefixedPath = $input->getArgument(self::PREFIXEDPATH); + + try { + $ch = curl_init($url); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + curl_setopt($ch, CURLOPT_TIMEOUT, 30); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_USERAGENT, self::USER_AGENT); + $content = curl_exec($ch); + if ($content && curl_getinfo($ch, CURLINFO_HTTP_CODE) === 200) { + $this->storageObjectManagement->uploadObject($content, [ + 'name' => $prefixedPath, + 'predefinedAcl' => $this->storageObjectManagement->getObjectAclPolicy() + ]); + } + curl_close($ch); + } catch (LocalizedException $e) { + $this->logger->error($e->getMessage()); + $exitCode = 1; + } + + return $exitCode; + } +} diff --git a/Console/Command/SynchronizeCommand.php b/Console/Command/SynchronizeCommand.php deleted file mode 100644 index 7749022..0000000 --- a/Console/Command/SynchronizeCommand.php +++ /dev/null @@ -1,121 +0,0 @@ - - * @license MIT - */ -declare(strict_types=1); - -namespace AuroraExtensions\GoogleCloudStorage\Console\Command; - -use Exception; -use AuroraExtensions\GoogleCloudStorage\Api\StorageTypeMetadataInterface; -use Magento\Framework\{ - App\Area, - App\State -}; -use Magento\MediaStorage\{ - Model\File\Storage, - Model\File\Storage\Flag -}; -use Psr\Log\LoggerInterface; -use Symfony\Component\{ - Console\Command\Command, - Console\Input\InputInterface, - Console\Output\OutputInterface -}; - -use const null; -use function strtotime; -use function time; - -class SynchronizeCommand extends Command -{ - /** @constant string COMMAND_NAME */ - private const COMMAND_NAME = 'gcs:media:sync'; - - /** @constant string COMMAND_DESC */ - private const COMMAND_DESC = 'Synchronize media storage with Google Cloud Storage.'; - - /** @var State $state */ - private $state; - - /** @var Storage $storage */ - private $storage; - - /** @var LoggerInterface $logger */ - private $logger; - - /** - * @param State $state - * @param Storage $storage - * @param LoggerInterface $logger - * @return void - */ - public function __construct( - State $state, - Storage $storage, - LoggerInterface $logger - ) { - $this->state = $state; - $this->storage = $storage; - $this->logger = $logger; - parent::__construct(); - } - - /** - * {@inheritdoc} - */ - protected function configure() - { - $this->setName(self::COMMAND_NAME); - $this->setDescription(self::COMMAND_DESC); - parent::configure(); - } - - /** - * {@inheritdoc} - */ - protected function execute(InputInterface $input, OutputInterface $output) - { - try { - $this->state->setAreaCode(Area::AREA_ADMINHTML); - - /** @var Flag $flag */ - $flag = $this->storage->getSyncFlag(); - - /** @var int|string|null $lastUpdate */ - $lastUpdate = $flag->getLastUpdate() ?: null; - - if ($flag->getState() === Flag::STATE_RUNNING && !empty($lastUpdate) && time() <= strtotime($lastUpdate) + Flag::FLAG_TTL) { - return; - } - - $flag->setState(Flag::STATE_RUNNING)->setFlagData([])->save(); - - try { - $this->storage->synchronize([ - 'type' => StorageTypeMetadataInterface::STORAGE_MEDIA_GCS, - ]); - } catch (Exception $e) { - $this->logger->critical($e); - $flag->passError($e); - } - - $flag->setState(Flag::STATE_FINISHED)->save(); - $output->writeln('Media synchronized successfully!'); - } catch (Exception $e) { - $output->writeln('Media synchronization failed!'); - } - } -} diff --git a/Exception/ExceptionFactory.php b/Exception/ExceptionFactory.php new file mode 100644 index 0000000..43fb6b5 --- /dev/null +++ b/Exception/ExceptionFactory.php @@ -0,0 +1,84 @@ + + * @license MIT + */ +declare(strict_types=1); + +namespace AuroraExtensions\GoogleCloudStorage\Exception; + +use Exception; +use Throwable; +use Magento\Framework\{ + ObjectManagerInterface, + Phrase +}; + +use function is_subclass_of; +use function __; + +class ExceptionFactory +{ + /** @constant string ERROR_DEFAULT_MSG */ + private const ERROR_DEFAULT_MSG = 'An error occurred. Unable to process the request.'; + + /** @constant string ERROR_INVALID_TYPE */ + private const ERROR_INVALID_TYPE = 'Invalid exception class type %1 was given.'; + + /** @var ObjectManagerInterface $objectManager */ + private $objectManager; + + /** + * @param ObjectManagerInterface $objectManager + * @return void + */ + public function __construct(ObjectManagerInterface $objectManager) + { + $this->objectManager = $objectManager; + } + + /** + * @param string|null $type + * @param Phrase|null $message + * @return Throwable + * @throws Exception + */ + public function create( + string $type = Exception::class, + Phrase $message = null + ) { + /** @var array $arguments */ + $arguments = []; + + /* Set default message, as required. */ + $message = $message ?? __(static::ERROR_DEFAULT_MSG); + + if (!is_subclass_of($type, Throwable::class)) { + throw new Exception( + __( + static::ERROR_INVALID_TYPE, + $type + )->__toString() + ); + } + + if ($type !== Exception::class) { + $arguments['phrase'] = $message; + } else { + $arguments['message'] = $message->__toString(); + } + + return $this->objectManager->create($type, $arguments); + } +} \ No newline at end of file diff --git a/Model/Adapter/StorageObjectManagement.php b/Model/Adapter/StorageObjectManagement.php index d039d48..d044bff 100644 --- a/Model/Adapter/StorageObjectManagement.php +++ b/Model/Adapter/StorageObjectManagement.php @@ -23,9 +23,8 @@ Api\StorageObjectPathResolverInterface, Component\ModuleConfigTrait, Exception\InvalidGoogleCloudStorageSetupException, - Model\System\ModuleConfig -}; -use AuroraExtensions\ModuleComponents\{ + Model\System\ModuleConfig, + Model\File\Storage, Api\LocalizedScopeDeploymentConfigInterface, Api\LocalizedScopeDeploymentConfigInterfaceFactory, Exception\ExceptionFactory @@ -41,6 +40,10 @@ Filesystem, Filesystem\Driver\File as FileDriver }; +use Magento\Store\Model\StoreManagerInterface; +use Magento\Framework\App\CacheInterface; +use Magento\Framework\Serialize\SerializerInterface; +use AuroraExtensions\GoogleCloudStorage\Model\Cache\Type\GcsCache; use Psr\Http\{ Message\StreamInterface, Message\StreamInterfaceFactory @@ -59,6 +62,8 @@ class StorageObjectManagement implements StorageObjectManagementInterface, StorageObjectPathResolverInterface { + const USER_AGENT = 'outeredge/gcs'; + /** * @var ModuleConfig $moduleConfig * @method ModuleConfig getConfig() @@ -92,6 +97,18 @@ class StorageObjectManagement implements StorageObjectManagementInterface, Stora /** @var bool $useModuleConfig */ private $useModuleConfig; + /** @var bool $enabled */ + private $enabled = false; + + /** @var StoreManagerInterface $storeManager */ + private $storeManager; + + /** @var CacheInterface $cache */ + private $cache; + + /** @var SerializerInterface $serializer */ + private $serializer; + /** * @param LocalizedScopeDeploymentConfigInterfaceFactory $deploymentConfigFactory * @param ExceptionFactory $exceptionFactory @@ -99,25 +116,35 @@ class StorageObjectManagement implements StorageObjectManagementInterface, Stora * @param Filesystem $filesystem * @param ModuleConfig $moduleConfig * @param StreamInterfaceFactory $streamFactory + * @param CacheInterface $cache + * @param SerializerInterface $serializer * @param bool $useModuleConfig * @return void */ public function __construct( LocalizedScopeDeploymentConfigInterfaceFactory $deploymentConfigFactory, ExceptionFactory $exceptionFactory, + Storage $fileStorage, FileDriver $fileDriver, Filesystem $filesystem, ModuleConfig $moduleConfig, StreamInterfaceFactory $streamFactory, + StoreManagerInterface $storeManager, + CacheInterface $cache, + SerializerInterface $serializer, bool $useModuleConfig = false ) { + $this->enabled = $fileStorage->checkBucketUsage(); $this->deploymentConfig = $deploymentConfigFactory->create(['scope' => 'googlecloud']); $this->exceptionFactory = $exceptionFactory; $this->fileDriver = $fileDriver; $this->filesystem = $filesystem; $this->moduleConfig = $moduleConfig; $this->streamFactory = $streamFactory; + $this->storeManager = $storeManager; $this->useModuleConfig = $useModuleConfig; + $this->cache = $cache; + $this->serializer = $serializer; $this->initialize(); } @@ -127,6 +154,10 @@ public function __construct( */ private function initialize(): void { + if (!$this->isEnabled()) { + return; + } + /** @var string|null $projectName */ $projectName = $this->useModuleConfig ? $this->getConfig()->getGoogleCloudProject() @@ -255,14 +286,63 @@ public function getClient(): StorageClient */ public function getObject(string $path): ?StorageObject { + $cache = $this->cache->load(GcsCache::TYPE_IDENTIFIER); + $cacheGcs = $cache ? $this->serializer->unserialize($cache) : []; + + if (in_array($path, $cacheGcs)) { + return null; + } + if ($this->hasPrefix()) { - $path = implode(DIRECTORY_SEPARATOR, [ + $prefixedPath = implode(DIRECTORY_SEPARATOR, [ $this->getPrefix(), ltrim($path, DIRECTORY_SEPARATOR), ]); } - return $this->bucket->object($path); + $object = $this->bucket->object($prefixedPath); + $fallback = $this->deploymentConfig->get('storage/fallback_url'); + $exists = false; + + if ($object->exists()) { + $exists = true; + } elseif ($fallback) { + $storecode = $this->storeManager->getStore()->getCode(); + + if ($storecode == 'admin' && stristr($_SERVER['REQUEST_URI'], '_admin')) { + $storecode = str_replace('_admin', '', explode('/', ltrim($_SERVER['REQUEST_URI'], '/'))[0]); + } + + if (is_array($fallback)) { + if (isset($_GET['imgstore']) && isset($fallback[$_GET['imgstore']])) { + $fallback = $fallback[$_GET['imgstore']]; + $storecode = $_GET['imgstore']; + } elseif (isset($fallback[$storecode])) { + $fallback = $fallback[$storecode]; + } else { + $fallback = $fallback['default']; + } + } + + /* Load the image in a shell background process */ + $url = $fallback . $path; + $cmd = sprintf( + 'MAGE_RUN_CODE=%s bin/magento outeredge:gcs:download %s %s', + escapeshellarg($storecode), + escapeshellarg($url), + escapeshellarg($prefixedPath) + ); + + shell_exec(sprintf('%s > /dev/null 2>&1 &', $cmd)); + } + + $this->cache->save( + $this->serializer->serialize(array_merge($cacheGcs, [$path => $exists])), + GcsCache::TYPE_IDENTIFIER, + [GcsCache::CACHE_TAG] + ); + + return $object; } /** @@ -292,6 +372,19 @@ public function getObjects(array $options = []): ?ObjectIterator */ public function objectExists(string $path): bool { + // Don't waste requests if path does not have an extension + if (strpos($path, '.') === false) { + return false; + } + + // Don't waste requests if an attempt has already been made + $cache = $this->cache->load(GcsCache::TYPE_IDENTIFIER); + $cacheGcs = $cache ? $this->serializer->unserialize($cache) : []; + + if (in_array($path, $cacheGcs)) { + return $cacheGcs[$path]; + } + /** @var StorageObject|null $object */ $object = $this->getObject($path); return ($object !== null && $object->exists()); @@ -311,9 +404,10 @@ public function uploadObject($handle, array $options = []): ?StorageObject $prefix = $this->getPrefix(); if (isset($options['name'])) { + $options['name'] = ltrim($options['name'], DIRECTORY_SEPARATOR); $options['name'] = implode(DIRECTORY_SEPARATOR, [ $prefix, - ltrim($options['name'], DIRECTORY_SEPARATOR . $prefix), + str_replace($prefix . DIRECTORY_SEPARATOR, '', $options['name']), ]); } else { /** @var StreamInterface $stream */ @@ -340,6 +434,10 @@ public function uploadObject($handle, array $options = []): ?StorageObject } } + if (stristr($options['name'], DIRECTORY_SEPARATOR . 'tmp' . DIRECTORY_SEPARATOR)) { + return null; + } + return $this->bucket->upload($handle, $options); } @@ -361,7 +459,7 @@ public function copyObject(string $source, string $target): ?StorageObject /** @var StorageObject $object */ $object = $this->getObject($source); - return $object->exists() ? $object->copy($target) : null; + return ($object !== null && $object->exists()) ? $object->copy($target) : null; } /** @@ -382,7 +480,7 @@ public function renameObject(string $source, string $target): ?StorageObject /** @var StorageObject $object */ $object = $this->getObject($source); - return $object->exists() ? $object->rename($target) : null; + return ($object !== null && $object->exists()) ? $object->rename($target) : null; } /** @@ -451,4 +549,12 @@ public function getObjectAclPolicy(): string return !empty($aclPolicy) ? $aclPolicy : ModuleConfig::DEFAULT_ACL_POLICY; } + + /** + * {@inheritdoc} + */ + public function isEnabled() : bool + { + return $this->enabled; + } } diff --git a/Model/Cache/Type/GcsCache.php b/Model/Cache/Type/GcsCache.php new file mode 100644 index 0000000..829967e --- /dev/null +++ b/Model/Cache/Type/GcsCache.php @@ -0,0 +1,24 @@ +get(self::TYPE_IDENTIFIER), + self::CACHE_TAG + ); + } +} diff --git a/Model/Config/Deployment/LocalizedScopeDeploymentConfig.php b/Model/Config/Deployment/LocalizedScopeDeploymentConfig.php new file mode 100644 index 0000000..8859e98 --- /dev/null +++ b/Model/Config/Deployment/LocalizedScopeDeploymentConfig.php @@ -0,0 +1,78 @@ + + * @license MIT + */ +declare(strict_types=1); + +namespace AuroraExtensions\GoogleCloudStorage\Model\Config\Deployment; + +use AuroraExtensions\GoogleCloudStorage\Api\LocalizedScopeDeploymentConfigInterface; +use Magento\Framework\App\DeploymentConfig; + +use function array_filter; +use function array_merge; +use function explode; +use function implode; +use function trim; + +class LocalizedScopeDeploymentConfig implements LocalizedScopeDeploymentConfigInterface +{ + /** @constant string DELIMITER */ + private const DELIMITER = '/'; + + /** @var DeploymentConfig $deploymentConfig */ + private $deploymentConfig; + + /** @var string $delimiter */ + private $delimiter; + + /** @var string $scope */ + private $scope; + + /** + * @param DeploymentConfig $deploymentConfig + * @param string $delimiter + * @param string|null $scope + * @return void + */ + public function __construct( + DeploymentConfig $deploymentConfig, + string $delimiter = self::DELIMITER, + string $scope = null + ) { + $this->deploymentConfig = $deploymentConfig; + $this->delimiter = $delimiter; + $this->scope = $scope ?? ''; + } + + /** + * {@inheritdoc} + */ + public function get(?string $path = null) + { + /** @var string $scope */ + $scope = trim($this->scope, $this->delimiter); + + /** @var array $parts */ + $parts = explode($this->delimiter, !empty($path) ? $path : ''); + + /** @var array $merge */ + $merge = array_merge([$scope], array_filter($parts, 'strlen')); + + /** @var string $xpath */ + $xpath = implode($this->delimiter, $merge); + return !empty($xpath) ? $this->deploymentConfig->get($xpath) : null; + } +} diff --git a/Model/Config/Source/Select/VirtualSelect.php b/Model/Config/Source/Select/VirtualSelect.php new file mode 100644 index 0000000..7bd3ebb --- /dev/null +++ b/Model/Config/Source/Select/VirtualSelect.php @@ -0,0 +1,71 @@ + + * @license MIT + */ +declare(strict_types=1); + +namespace AuroraExtensions\GoogleCloudStorage\Model\Config\Source\Select; + +use Magento\Framework\Data\OptionSourceInterface; + +use const true; +use function array_flip; +use function array_walk; +use function __; + +class VirtualSelect implements OptionSourceInterface +{ + /** @var array $options */ + private $options = []; + + /** + * @param array $data + * @param bool $flip + * @return void + */ + public function __construct( + array $data = [], + bool $flip = true + ) { + /** @var array $opts */ + $opts = $flip ? array_flip($data) : $data; + + array_walk($opts, [ + $this, + 'setOption' + ]); + } + + /** + * @param int|string|null $value + * @param int|string $key + * @return void + */ + private function setOption($value, $key): void + { + $this->options[] = [ + 'label' => __($key), + 'value' => $value, + ]; + } + + /** + * @return array + */ + public function toOptionArray() + { + return $this->options; + } +} \ No newline at end of file diff --git a/Model/File/Storage/Bucket.php b/Model/File/Storage/Bucket.php index bc6197e..638dbfe 100644 --- a/Model/File/Storage/Bucket.php +++ b/Model/File/Storage/Bucket.php @@ -23,9 +23,9 @@ Api\StorageObjectManagementInterface, Component\ModuleConfigTrait, Component\StorageAdapterTrait, - Model\System\ModuleConfig + Model\System\ModuleConfig, + Exception\ExceptionFactory }; -use AuroraExtensions\ModuleComponents\Exception\ExceptionFactory; use Google\Cloud\{ Storage\StorageObject, Storage\ObjectIterator @@ -33,6 +33,7 @@ use Magento\Framework\{ App\Filesystem\DirectoryList, Exception\LocalizedException, + Exception\FileSystemException, Filesystem, Filesystem\Driver\File as FileDriver, Model\AbstractModel, @@ -149,18 +150,15 @@ public function setObjects(?ObjectIterator $objects) } /** - * @param string $filename + * @param string $relativePath * @return $this */ - public function loadByFilename(string $filename) + public function loadByFilename(string $relativePath) { - /** @var string $relativePath */ - $relativePath = $this->storageHelper->getMediaRelativePath($filename); - - if ($this->getStorage()->objectExists($relativePath)) { - $this->setData('id', $filename); - $this->setData('filename', $filename); - $this->setData('content', $this->getStorage()->getObject($relativePath)->downloadAsString()); + if ($object = $this->getStorage()->getObject($relativePath)) { + $this->setData('id', $relativePath); + $this->setData('filename', $relativePath); + $this->setData('content', $object->downloadAsString()); } else { $this->unsetData(); } @@ -331,6 +329,44 @@ public function saveFile(string $filename) return $this; } + /** + * @param string $filePath + * @return bool Returns true on existing file or successful download, false on failure + */ + public function downloadFile(string $filePath): bool + { + $relativePath = $this->storageHelper->getMediaRelativePath($filePath); + // @todo look up media path like https://github.com/magento/magento2/blob/2.4-develop/app/code/Magento/MediaStorage/App/Media.php#L187 + $relativePath = str_replace(DirectoryList::MEDIA . DIRECTORY_SEPARATOR, '', $relativePath); + + $mediaPath = $this->filesystem->getDirectoryWrite(DirectoryList::MEDIA); + + if ($mediaPath->isFile($relativePath)) { + return true; + } + + try { + $this->loadByFilename($relativePath); + } catch (\Exception $e) { + return false; + } + + if ($this->getId()) { + $file = $mediaPath->openFile($relativePath, 'w'); + try { + $file->lock(); + $file->write($this->getContent()); + $file->unlock(); + $file->close(); + return true; + } catch (FileSystemException $e) { + $file->close(); + } + } + + return false; + } + /** * @param string $filePath * @return bool diff --git a/Model/File/Storage/Synchronization.php b/Model/File/Storage/Synchronization.php new file mode 100644 index 0000000..5576a83 --- /dev/null +++ b/Model/File/Storage/Synchronization.php @@ -0,0 +1,35 @@ +storageFactory = $storageFactory; + } + + /** + * Synchronize file from GCS to local filesystem + * + * @param string $relativeFileName + * @return void + */ + public function synchronize($relativeFileName) + { + /** @var $storage Bucket */ + $storage = $this->storageFactory->create(); + + if (!$storage->getStorage()->isEnabled()) { + return; + } + + $storage->downloadFile($relativeFileName); + } +} diff --git a/Model/Utils/PathUtils.php b/Model/Utils/PathUtils.php new file mode 100644 index 0000000..813c717 --- /dev/null +++ b/Model/Utils/PathUtils.php @@ -0,0 +1,201 @@ + + * @license MIT + */ +declare(strict_types=1); + +namespace AuroraExtensions\GoogleCloudStorage\Model\Utils; + +use Magento\Framework\Stdlib\StringUtils; + +use const DIRECTORY_SEPARATOR; +use const PHP_MAXPATHLEN; +use function array_filter; +use function array_map; +use function array_merge; +use function array_pop; +use function array_shift; +use function array_values; +use function explode; +use function implode; + +class PathUtils +{ + /** @constant int TRIM_LEFT */ + public const TRIM_LEFT = 1; + + /** @constant int TRIM_RIGHT */ + public const TRIM_RIGHT = 2; + + /** @constant array TRIM_CONTEXTS */ + public const TRIM_CONTEXTS = [ + self::TRIM_LEFT => 'ltrim', + self::TRIM_RIGHT => 'rtrim', + self::TRIM_LEFT | self::TRIM_RIGHT => 'trim', + ]; + + /** @var StringUtils $stringUtils */ + private $stringUtils; + + /** + * @param StringUtils $stringUtils + * @return void + */ + public function __construct(StringUtils $stringUtils) + { + $this->stringUtils = $stringUtils; + } + + /** + * @param string[] $pieces + * @return string + */ + public function build(string ...$pieces): string + { + /** @var string $basePath */ + $basePath = $this->trim((string) array_shift($pieces), self::TRIM_RIGHT); + + if (!empty($basePath)) { + /** @var bool $isAbsolute */ + $isAbsolute = false; + + if ($basePath[0] === DIRECTORY_SEPARATOR) { + $isAbsolute = true; + } + + /** @var string[] $baseParts */ + $baseParts = $this->stringUtils->split($basePath, PHP_MAXPATHLEN, true, true, '\/'); + + /** @var string[] $basePaths */ + $basePaths = $this->split(!empty($baseParts) ? $baseParts[0] : $basePath, ' ', true); + $basePath = $this->concat($isAbsolute ? array_merge([null], $basePaths) : $basePaths); + } + + /** @var array $result */ + $result[] = [$basePath]; + + /** @var string $basename */ + $basename = $this->trim((string) array_pop($pieces)); + + if (!empty($basename)) { + /** @var string[] $nameParts */ + $nameParts = $this->split($basename, DIRECTORY_SEPARATOR, true); + $basename = $this->concat($nameParts); + } + + /** @var string[] $dirs */ + $dirs = $this->filter($pieces); + + /** @var int|string $key */ + /** @var string $value */ + foreach ($dirs as $key => $value) { + /** @var string[] $parts */ + $parts = $this->stringUtils->split($value, PHP_MAXPATHLEN, true, true, '\/'); + + /** @var string[] $paths */ + $paths = $this->split(!empty($parts) ? $parts[0] : $value, ' ', true); + + /** @var string $path */ + $path = $this->concat($paths); + $dirs[$key] = $this->trim($path); + } + + $result[] = $dirs; + $result[] = [$basename]; + + return $this->concat(array_merge(...$result)); + } + + /** + * @param array $pieces + * @param string $delimiter + * @param bool $filter + * @return string + */ + public function concat( + array $pieces, + string $delimiter = DIRECTORY_SEPARATOR, + bool $filter = false + ): string { + if ($filter) { + $pieces = array_filter($pieces, 'strlen'); + } + + return implode($delimiter, $pieces); + } + + /** + * @param array $pieces + * @param callable|null $callback + * @param bool $preserveKeys + * @return array + */ + public function filter( + array $pieces, + callable $callback = null, + bool $preserveKeys = false + ): array { + /* Defaults strlen for empty values. */ + $callback = $callback ?? 'strlen'; + + /** @var array $result */ + $result = array_filter($pieces, $callback); + return $preserveKeys ? $result : array_values($result); + } + + /** + * @param string $subject + * @param string $delimiter + * @param bool $filter + * @return string[] + */ + public function split( + string $subject, + string $delimiter = DIRECTORY_SEPARATOR, + bool $filter = false + ): array { + /** @var string[] $pieces */ + $pieces = explode($delimiter, $subject); + + if ($filter) { + $pieces = array_filter($pieces, 'strlen'); + } + + return array_values($pieces); + } + + /** + * @param string $subject + * @param int $context + * @param string $delimiter + * @return string + */ + public function trim( + string $subject, + int $context = self::TRIM_LEFT | self::TRIM_RIGHT, + string $delimiter = DIRECTORY_SEPARATOR + ): string { + /** @var string|null $callback */ + $callback = self::TRIM_CONTEXTS[$context] ?? null; + + if ($callback === null) { + return $subject; + } + + /** @var array $result */ + $result = array_map($callback, [$subject], [$delimiter]); + return !empty($result) ? $result[0] : $subject; + } +} \ No newline at end of file diff --git a/Plugin/Catalog/Category.php b/Plugin/Catalog/Category.php new file mode 100644 index 0000000..8f21524 --- /dev/null +++ b/Plugin/Catalog/Category.php @@ -0,0 +1,26 @@ +synchronization = $synchronization; + } + + public function afterGetImageUrl(CategoryModel $category, $result) + { + $this->synchronization->synchronize($result); + return $result; + } +} diff --git a/Plugin/Catalog/Product/Gallery/ObjectUploader.php b/Plugin/Catalog/Product/Gallery/ObjectUploader.php index 514f4d0..304d9d1 100644 --- a/Plugin/Catalog/Product/Gallery/ObjectUploader.php +++ b/Plugin/Catalog/Product/Gallery/ObjectUploader.php @@ -91,12 +91,16 @@ public function __construct( /** * @param ExtensionInterface $subject * @param ProductInterface $result - * @return void + * @return ProductInterface */ public function afterExecute( ExtensionInterface $subject, ProductInterface $result - ) { + ): ProductInterface { + if (!$this->storageAdapter->isEnabled()) { + return $result; + } + /** @var string $attrCode */ $attrCode = $subject->getAttribute() ->getAttributeCode(); diff --git a/Plugin/Catalog/Product/Image.php b/Plugin/Catalog/Product/Image.php new file mode 100644 index 0000000..5a68af0 --- /dev/null +++ b/Plugin/Catalog/Product/Image.php @@ -0,0 +1,33 @@ +synchronization = $synchronization; + $this->mediaConfig = $mediaConfig; + } + + public function beforeSetBaseFile(ProductImage $image, $file) + { + $this->synchronization->synchronize($this->mediaConfig->getBaseMediaPath() . $file); + } +} diff --git a/Plugin/Catalog/View/Asset/Image.php b/Plugin/Catalog/View/Asset/Image.php new file mode 100644 index 0000000..540bd5e --- /dev/null +++ b/Plugin/Catalog/View/Asset/Image.php @@ -0,0 +1,56 @@ +mediaConfig = $mediaConfig; + $this->storageFactory = $storageFactory; + $this->imageHelper = $imageHelper; + } + + public function beforeGetUrl(LocalInterface $image) + { + /** @var $storage Storage\Bucket */ + $storage = $this->storageFactory->create(); + + if (!$storage->getStorage()->isEnabled()) { + return; + } + + // Download the main image + $relativeFileName = $this->mediaConfig->getBaseMediaPath() . $image->getFilePath(); + $storage->downloadFile($relativeFileName); + + // Download the cached image + if ($image->getModule() == 'cache') { + $cacheFilename = $this->imageHelper->prepareFilename($image->getPath()); + $storage->downloadFile($cacheFilename); + } + } +} diff --git a/Plugin/Framework/File/Uploader/ObjectUploader.php b/Plugin/Framework/File/Uploader/ObjectUploader.php index 6920519..9f75bcb 100644 --- a/Plugin/Framework/File/Uploader/ObjectUploader.php +++ b/Plugin/Framework/File/Uploader/ObjectUploader.php @@ -23,9 +23,9 @@ Api\StorageObjectManagementInterface, Component\ModuleConfigTrait, Component\StorageAdapterTrait, - Model\System\ModuleConfig + Model\System\ModuleConfig, + Model\Utils\PathUtils }; -use AuroraExtensions\ModuleComponents\Model\Utils\PathUtils; use Magento\Framework\{ Exception\FileSystemException, File\Uploader, @@ -95,6 +95,10 @@ public function afterSave( $destinationFolder, $newFileName = null ) { + if (!$this->storageAdapter->isEnabled()) { + return $result; + } + if (!empty($result)) { /** @var string $basePath */ $basePath = (string)($result['path'] ?? ''); diff --git a/Plugin/Framework/Image/Adapter/ObjectAdapter.php b/Plugin/Framework/Image/Adapter/ObjectAdapter.php index 23f5b33..a9d8285 100644 --- a/Plugin/Framework/Image/Adapter/ObjectAdapter.php +++ b/Plugin/Framework/Image/Adapter/ObjectAdapter.php @@ -90,6 +90,10 @@ public function afterSave( $destination = null, $newName = null ) { + if (!$this->storageAdapter->isEnabled()) { + return; + } + if (!empty($destination)) { /** @var string $filePath */ $filePath = $this->storageHelper->getMediaRelativePath($destination); diff --git a/Plugin/MediaStorage/File/Storage/FileProcessor.php b/Plugin/MediaStorage/File/Storage/FileProcessor.php index 2535465..5396f91 100644 --- a/Plugin/MediaStorage/File/Storage/FileProcessor.php +++ b/Plugin/MediaStorage/File/Storage/FileProcessor.php @@ -73,7 +73,7 @@ public function afterSaveFile( $file, $overwrite = true ) { - if (!$result) { + if (!$result || !$this->storageAdapter->isEnabled()) { return $result; } diff --git a/README.rst b/README.rst deleted file mode 100644 index c95e41f..0000000 --- a/README.rst +++ /dev/null @@ -1,85 +0,0 @@ -Google Cloud Storage -==================== - -.. contents:: - :local: - -Description ------------ - -.. |link1| replace:: Google Cloud Storage -.. |link2| replace:: Google Cloud CDN -.. |link3| replace:: Documentation -.. |link4| replace:: Creating and Managing Service Account Keys -.. |link5| replace:: env.php.sample -.. _link1: https://cloud.google.com/storage/ -.. _link2: https://cloud.google.com/cdn/ -.. _link3: https://docs.auroraextensions.com/magento/extensions/2.x/googlecloudstorage/latest/index.html -.. _link4: https://cloud.google.com/iam/docs/creating-managing-service-account-keys -.. _link5: https://github.com/auroraextensions/googlecloudstorage/blob/master/env.php.sample - -Use |link1|_ to store media assets in Magento. - -Installation ------------- - -We highly recommend installing via Composer for package management. - -.. code-block:: sh - - composer require auroraextensions/googlecloudstorage - -Configuration -------------- - -Once installed, update the environment configuration file. See |link5|_ for an example. -The following information should be readily available: - -1. Google Cloud project ID -2. Path to the Google Cloud service account JSON key file. See `Service Account`_ for more details. -3. Google Cloud Storage bucket name -4. Google Cloud Storage bucket region (if applicable) - -Next, enable the module with the Magento autoloader. - -.. code-block:: sh - - php bin/magento module:enable AuroraExtensions_GoogleCloudStorage - -Synchronization ---------------- - -You can initiate the bulk synchronization process through the Magento backend, just as you would with -any other media storage configuration. Additionally, you can initiate the bulk synchronization process -from the command line using the provided synchronization CLI command. - -.. code-block:: sh - - php bin/magento gcs:media:sync - -**IMPORTANT**: This process can be very slow, especially if you have a lot of media files. - -Service Account ---------------- - -For the purposes of authenticating with Google Cloud Platform, this module leverages the flexibility and ease of use provided by Google Cloud service accounts. -Before moving forward, please make sure to complete the following: - -1. Create a Google Cloud service account with **Storage Admin** privileges. Once the service account is created, you will be prompted to download a JSON key file. Store this key file in a safe place. -2. Install the service account JSON key file to the local or mounted filesystem with read-only permissions for the Magento user. -3. Verify the following fields are set and correct in the environment configuration file (env.php): - 1. All required fields - 2. The Google Cloud project name is where the bucket exists - 3. The path to the Google Cloud service account JSON key file (e.g. /etc/gcs.json). Relative paths are assumed to be relative to the Magento root directory. - 4. [OPTIONAL] If you use the same bucket for multiple projects, you can specify a subdirectory to synchronize to inside the bucket. By default, it will synchronize to /. - -For more information on Google Cloud service account keys, please see |link4|_. - -For an example configuration file, please see |link5|_. - -Troubleshooting ---------------- - - Given keyfile at path /path/to/magento was invalid - -You need to create and install a service account key to authenticate with Google Cloud. See `Service Account`_ for specific details on Google Cloud service accounts. diff --git a/composer.json b/composer.json index 291c604..7efbdbb 100644 --- a/composer.json +++ b/composer.json @@ -1,29 +1,16 @@ { - "name": "auroraextensions/googlecloudstorage", + "name": "outeredge/googlecloudstorage", "description": "Google Cloud Storage integration for Magento.", "type": "magento2-module", "license": "MIT", - "authors": [ - { - "name": "Nickolas Burr", - "email": "nickolasburr@auroraextensions.com" - } - ], - "support": { - "email": "support@auroraextensions.com" - }, "require": { - "php": "~7.2.0||~7.3.0||~7.4.0", - "psr/http-message": "~1.0", - "psr/log": "~1.0", - "symfony/console": "^2.7||^3.0||^4.0||^5.0", - "google/crc32": "~0.1.0", - "google/cloud-core": "~1.33.0", - "google/cloud-storage": "~1.14.0", - "magento/framework": "^100||^101||^102||^103", - "magento/module-media-storage": "^100||^101", - "magento/module-store": "^100||^101", - "auroraextensions/modulecomponents": "~100.0.0||~100.1.0" + "psr/http-message": ">=1.0", + "psr/log": ">=1.0", + "google/cloud-storage": "~1.27.0", + "magento/framework": "^103", + "magento/module-media-storage": "^100", + "magento/module-store": "^101", + "outeredge/magento-base-module": ">=2.6.33" }, "autoload": { "files": [ diff --git a/env.php.sample b/env.php.sample index 8c42866..7567747 100644 --- a/env.php.sample +++ b/env.php.sample @@ -10,6 +10,7 @@ return [ 'acl' => 'publicRead', 'region' => 'us-east1', ], + 'fallback_url' => 'https://example.com/media/' ], ], ]; diff --git a/etc/cache.xml b/etc/cache.xml new file mode 100644 index 0000000..91da8fb --- /dev/null +++ b/etc/cache.xml @@ -0,0 +1,7 @@ + + + + + Cache for saving GCS path + + diff --git a/etc/di.xml b/etc/di.xml index 8089cd6..071c7a7 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -17,11 +17,17 @@ */ --> + + + + + type="AuroraExtensions\GoogleCloudStorage\Model\Config\Source\Select\VirtualSelect"> Authenticated Read @@ -35,7 +41,7 @@ + type="AuroraExtensions\GoogleCloudStorage\Model\Config\Source\Select\VirtualSelect"> Multi-region (multi-region) @@ -60,13 +66,6 @@ - - - - AuroraExtensions\GoogleCloudStorage\Console\Command\SynchronizeCommand - - - + + + + + + + + + + + + + + + + AuroraExtensions\GoogleCloudStorage\Console\Command\DownloadImage + + + diff --git a/etc/module.xml b/etc/module.xml index aa9b072..d541b7b 100644 --- a/etc/module.xml +++ b/etc/module.xml @@ -19,7 +19,7 @@ - + diff --git a/view/frontend/layout/default.xml b/view/frontend/layout/default.xml new file mode 100644 index 0000000..fecc883 --- /dev/null +++ b/view/frontend/layout/default.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/view/frontend/templates/image-catcher.phtml b/view/frontend/templates/image-catcher.phtml new file mode 100644 index 0000000..e488ff8 --- /dev/null +++ b/view/frontend/templates/image-catcher.phtml @@ -0,0 +1,78 @@ +isEnabled()): ?> + +