diff --git a/apps/Core/Components/Devtools/Modules/ModulesComponent.php b/apps/Core/Components/Devtools/Modules/ModulesComponent.php index bf28933ee..15a1228bf 100644 --- a/apps/Core/Components/Devtools/Modules/ModulesComponent.php +++ b/apps/Core/Components/Devtools/Modules/ModulesComponent.php @@ -3,6 +3,10 @@ namespace Apps\Core\Components\Devtools\Modules; use Apps\Core\Packages\Devtools\Modules\DevtoolsModules; +use League\Flysystem\FilesystemException; +use League\Flysystem\UnableToCheckExistence; +use League\Flysystem\UnableToListContents; +use League\Flysystem\UnableToRetrieveMetadata; use System\Base\BaseComponent; use z4kn4fein\SemVer\Version; @@ -176,6 +180,30 @@ public function viewAction() unset($modules['views']); } } + } else if (isset($this->getData()['changes']) && + $this->getData()['changes'] == true + ) { + unset($modules['core']); + unset($modules['apptypes']); + unset($modules['bundles']); + + foreach ($modules as $moduleType => &$modulesTypeArr) { + if (isset($modulesTypeArr['childs']) && count($modulesTypeArr['childs']) > 0) { + foreach ($modulesTypeArr['childs'] as $childKey => &$child) { + $child = $this->modulesPackage->validateFilesHash($child); + + if ($child['repoExists'] && !$child['isModified']) { + unset($modules[$moduleType]['childs'][$childKey]); + } + } + } + } + + $this->view->modules = $modules; + + $this->view->pick('modules/changes'); + + return; } $this->view->modules = $modules; diff --git a/apps/Core/Components/Devtools/Test/Install/component.json b/apps/Core/Components/Devtools/Test/Install/component.json index f05292a1b..7e60a9450 100644 --- a/apps/Core/Components/Devtools/Test/Install/component.json +++ b/apps/Core/Components/Devtools/Test/Install/component.json @@ -1,45 +1,58 @@ { - "route" : "devtools/test", - "name" : "Test", - "description" : "Test anything in this component.", - "module_type" : "components", - "app_type" : "core", - "category" : "devtools", - "version" : "0.0.0", - "repo" : "https://.../dash-component-system-devtools-test", - "class" : "Apps\\Core\\Components\\Devtools\\Test\\TestComponent", - "dependencies" : { - "core" : { - "name" : "Core", - "version" : "0.0.0", - "repo" : "https://.../" - }, - "components" : [], - "packages" : - [ - { - "name" : "Test", - "version" : "0.0.0", - "repo" : "https://.../dash-package-system-devtools-test" - } - ], - "middlewares" : [], - "views" : [], - "external" : [] - }, - "menu" : { - "seq" : 99, - "devtools" : { - "title" : "devtools", - "icon" : "code", - "childs" : { - "test" : { - "title" : "test", - "link" : "devtools/test" - } - } - } - }, - "settings" : { - } + "name" : "Test", + "route" : "devtools/test", + "description" : "Test anything in this component.", + "module_type" : "components", + "app_type" : "core", + "category" : "devtools", + "version" : "0.0.0", + "repo" : "https://.../dash-component-system-devtools-test", + "class" : "Apps\\Core\\Components\\Devtools\\Test\\TestComponent", + "dependencies" : + { + "core" : + { + "name" : "Core", + "version" : "0.0.0", + "repo" : "https://.../" + }, + "apptype" : [], + "packages" : [], + "middlewares" : [], + "views" : [], + "externals" : + { + "composer" : + { + "require" : [] + }, + "config" : + { + "allow-plugins" : [] + }, + "extra" : + { + "patches" : [] + } + } + }, + "menu" : + { + "seq" : 99, + "devtools" : + { + "title" : "devtools", + "icon" : "code", + "childs" : + { + "test" : + { + "title" : "test", + "link" : "devtools/test" + } + } + } + }, + "settings" : [], + "widgets" : [] } \ No newline at end of file diff --git a/apps/Core/Components/Devtools/Test/TestComponent.php b/apps/Core/Components/Devtools/Test/TestComponent.php index 4a34c9334..83d76d486 100644 --- a/apps/Core/Components/Devtools/Test/TestComponent.php +++ b/apps/Core/Components/Devtools/Test/TestComponent.php @@ -3,37 +3,24 @@ namespace Apps\Core\Components\Devtools\Test; use Apps\Core\Packages\Devtools\DicExtractData\DevtoolsDicExtractData; +use Apps\Core\Packages\Devtools\Test\DevtoolsTest; use System\Base\BaseComponent; class TestComponent extends BaseComponent { + protected $testPackage; + + public function initialize() + { + $this->testPackage = new DevtoolsTest; + } + /** * @acl(name=view) */ public function viewAction() { - // $install = new \Apps\Fintech\Components\Dashboards\Install\Install; - - // $install->init()->install(); - // $adminComponents = $this->basepackages->utils->scanDir('apps/Core/Components/', true); - - // foreach ($adminComponents['files'] as $adminComponentKey => $adminComponent) { - // if (strpos($adminComponent, 'component.json')) { - // try { - // $jsonFile = - // $this->helper->decode( - // $this->localContent->read($adminComponent), - // true - // ); - // } catch (\throwable $e) { - // throw new \Exception($e->getMessage() . '. Problem reading component.json at location ' . $adminComponent); - // } - - // if ($jsonFile['menu'] && $jsonFile['menu'] !== 'false') { - // $this->basepackages->menus->addMenu($jsonFile); - // } - // } - // } + return; } /** @@ -82,4 +69,48 @@ public function apiRemoveAction() { $this->addResponse('Test', 0, ['remove' => true]); } + + public function testAction() + { + if ($this->basepackages->progress->checkProgressFile()) { + $this->basepackages->progress->deleteProgressFile(); + } + + $this->basepackages->progress->registerMethods( + [ + [ + 'method' => 'testTest', + 'text' => 'Test', + ], + [ + 'method' => 'testTest', + 'text' => 'Test', + ], + [ + 'method' => 'testTest', + 'text' => 'Test', + ], + [ + 'method' => 'testDownload', + 'text' => 'Download Data...', + 'remoteWeb' => true + ], + // [ + // 'method' => 'testProcess', + // 'text' => 'Process Data...', + // 'steps' => true + // ] + ] + ); + + $this->testPackage->testTest(); + $this->testPackage->testTest(); + $this->testPackage->testTest(); + $this->testPackage->testDownload(); + + $this->addResponse( + $this->testPackage->packagesData->responseMessage, + $this->testPackage->packagesData->responseCode + ); + } } \ No newline at end of file diff --git a/apps/Core/Components/Register/RegisterComponent.php b/apps/Core/Components/Register/RegisterComponent.php index 3de58fabf..26678dde7 100644 --- a/apps/Core/Components/Register/RegisterComponent.php +++ b/apps/Core/Components/Register/RegisterComponent.php @@ -114,6 +114,8 @@ public function viewAction() return; } + $this->view->refresh = false; + if (isset($this->getData()['api'])) { $api = $this->api->getById($this->getData()['api']); @@ -144,8 +146,6 @@ public function viewAction() $this->response->setStatusCode(404); return $this->response->send(); - - exit; } $this->view->setLayout('auth'); diff --git a/apps/Core/Components/System/Api/Client/Services/ServicesComponent.php b/apps/Core/Components/System/Api/Client/Services/ServicesComponent.php index 8c161be78..fe096f34f 100644 --- a/apps/Core/Components/System/Api/Client/Services/ServicesComponent.php +++ b/apps/Core/Components/System/Api/Client/Services/ServicesComponent.php @@ -27,6 +27,10 @@ public function viewAction() $this->view->apiLocations = $this->apiPackage->apiLocations; if (isset($this->getData()['id'])) { + if (isset($this->getData()['clone'])) { + $this->view->clone = true; + } + if ($this->getData()['id'] != 0) { $api = $this->apiPackage->getApiById($this->getData()['id']); @@ -96,6 +100,7 @@ function ($dataArr) { 'actionsToEnable' => [ 'edit' => 'system/api/client/services', + 'clone' => 'system/api/client/services', 'remove' => 'system/api/client/services/remove' ] ]; diff --git a/apps/Core/Components/System/Progress/ProgressComponent.php b/apps/Core/Components/System/Progress/ProgressComponent.php index 057994c09..3ccb8be57 100644 --- a/apps/Core/Components/System/Progress/ProgressComponent.php +++ b/apps/Core/Components/System/Progress/ProgressComponent.php @@ -33,4 +33,32 @@ public function getProgressAction() ); } } + + public function cancelProgressAction() + { + $this->requestIsPost(); + + $fileName = $this->session->getId(); + + if (isset($this->postData()['session'])) { + $fileName = $this->postData()['session']; + } else if (isset($this->postData()['file_name'])) { + $fileName = $this->postData()['file_name']; + } + + $progress = $this->basepackages->progress->cancelProgress($fileName); + + if ($progress) { + $this->addResponse( + $this->basepackages->progress->packagesData->responseMessage, + $this->basepackages->progress->packagesData->responseCode, + $progress, + ); + } else { + $this->addResponse( + $this->basepackages->progress->packagesData->responseMessage, + $this->basepackages->progress->packagesData->responseCode + ); + } + } } \ No newline at end of file diff --git a/apps/Core/Packages/Adminltetags/Tags/Content/Listing/Table/DynamicTable.php b/apps/Core/Packages/Adminltetags/Tags/Content/Listing/Table/DynamicTable.php index 621ea5a61..e652bd56e 100644 --- a/apps/Core/Packages/Adminltetags/Tags/Content/Listing/Table/DynamicTable.php +++ b/apps/Core/Packages/Adminltetags/Tags/Content/Listing/Table/DynamicTable.php @@ -368,14 +368,12 @@ protected function generateRowsContent() } foreach ($column as $controlKey => $control) { - $this->dtParams['dtControlsLinkClass'] = isset($this->params['dtControlsLinkClass']) ? $this->params['dtControlsLinkClass'] : 'contentAjaxLink'; if ($controlKey === 'view') { - if (is_array($control)) { $control = $control['link']; $title = @@ -410,7 +408,6 @@ protected function generateRowsContent() ] ); } else if ($controlKey === 'edit') { - if (is_array($control)) { $control = $control['link']; $title = @@ -444,8 +441,41 @@ protected function generateRowsContent() ] ] ); - } else if ($controlKey === 'remove') { + } else if ($controlKey === 'clone') { + if (is_array($control)) { + $control = $control['link']; + $title = + isset($control['title']) ? + strtoupper($control['title']) : + 'CLONE'; + $icon = + isset($control['icon']) ? + strtoupper($control['icon']) : + 'clone'; + $type = + isset($control['type']) ? + strtoupper($control['type']) : + 'primary'; + } else { + $title = 'CLONE'; + $icon = 'clone'; + $type = 'primary'; + } + + $controlButtons = array_merge($controlButtons, + [ + $controlKey => + [ + 'title' => $title, + 'additionalClass' => 'rowEdit ' . $this->dtParams['dtControlsLinkClass'], + 'icon' => $icon, + 'buttonType' => $type, + 'link' => $control + ] + ] + ); + } else if ($controlKey === 'remove') { if (is_array($control)) { $control = $control['link']; $title = diff --git a/apps/Core/Packages/Adminltetags/Traits/DynamicTable.php b/apps/Core/Packages/Adminltetags/Traits/DynamicTable.php index bb9c8488e..b004cf455 100644 --- a/apps/Core/Packages/Adminltetags/Traits/DynamicTable.php +++ b/apps/Core/Packages/Adminltetags/Traits/DynamicTable.php @@ -219,6 +219,10 @@ public function generateDTContent( $actions[$key] = $this->links->url($action . '/q/id/' . $row['id']); } } + + if ($key === 'clone') { + $actions[$key] = $actions[$key] . '/clone/true'; + } } $row["__control"] = $actions; diff --git a/apps/Core/Packages/Devtools/Modules/DevtoolsModules.php b/apps/Core/Packages/Devtools/Modules/DevtoolsModules.php index 767af93b7..024acaa05 100644 --- a/apps/Core/Packages/Devtools/Modules/DevtoolsModules.php +++ b/apps/Core/Packages/Devtools/Modules/DevtoolsModules.php @@ -2,6 +2,7 @@ namespace Apps\Core\Packages\Devtools\Modules; +use Apps\Core\Packages\Devtools\Modules\Model\AppsCoreDevtoolsFilesHash; use Apps\Core\Packages\Devtools\Modules\Settings; use League\Flysystem\FilesystemException; use League\Flysystem\UnableToCheckExistence; @@ -20,6 +21,8 @@ class DevtoolsModules extends BasePackage { + protected $modelToUse = AppsCoreDevtoolsFilesHash::class; + protected $apiClient; protected $apiClientConfig; @@ -146,6 +149,10 @@ public function addModule($data) $this->updateModuleJson($data, false, $viewPublic) && $this->modules->{$data['module_type']}->add($data) ) { + if (strtolower($data['app_type']) !== 'core') { + $this->reCalculateFilesHash($this->modules->{$data['module_type']}->packagesData->last); + } + if ($data['createrepo'] == true) { if ($data['module_type'] === 'views' && $data['is_subview'] == false) {//Create public repository as well if (!$this->checkRepo($data)) { @@ -292,6 +299,10 @@ public function updateModule($data) if ($this->updateModuleJson($data, false, $viewPublic) && $this->modules->{$data['module_type']}->update($module) ) { + if (strtolower($data['app_type']) !== 'core') { + $this->reCalculateFilesHash($this->modules->{$data['module_type']}->packagesData->last); + } + if ($data['module_type'] === 'components') { $this->addUpdateComponentMenu($data); } @@ -448,6 +459,10 @@ public function removeModule($data) $this->modules->{$module['module_type']}->remove($module['id']); } + if ($data['module_type'] !== 'bundles') { + $this->reCalculateFilesHash($module, true); + } + $this->addResponse('Removed module from DB & files from the system...'); return true; @@ -458,6 +473,235 @@ public function removeModule($data) } } + protected function getFilesHash($module) + { + if ($this->config->databasetype === 'db') { + $conditions = + [ + 'conditions' => 'module_type = :module_type: AND module_id = :module_id:', + 'bind' => + [ + 'module_type' => $module['module_type'], + 'module_id' => $module['id'] + ] + ]; + } else { + $conditions = + [ + 'conditions' => [ + ['module_type', '=', $module['module_type']], + ['module_id', '=', $module['id']] + ] + ]; + } + + $filesHash = $this->getByParams($conditions); + if ($filesHash && isset($filesHash[0])) { + $filesHash = $filesHash[0]; + } + + return $filesHash; + } + + public function reCalculateFilesHash($module, $remove = false, $viaGenerateRelease = false, $viaValidation = false) + { + if (!$viaValidation) { + $filesHash = $this->getFilesHash($module); + + if ($filesHash && $remove) { + $this->remove($filesHash['id']); + + return true; + } + } else { + $filesHash = false; + } + + if (!$filesHash || $viaGenerateRelease) {//We only generate hash when there is no entry or when we generate a new release + if (!$viaGenerateRelease) { + $filesHash = []; + $filesHash['module_type'] = $module['module_type']; + $filesHash['module_id'] = $module['id']; + } + + $moduleLocation = $this->getModuleFilesLocation($module); + + $moduleLocationFiles = $this->basepackages->utils->scanDir($moduleLocation, true, ['.git/', '.fileHashes']); + + //In case you have .gitignore file, it can get complicated to sort which files to hash. + //So, the files you want to hash, just add them in .fileHashes in the same folder as .gitignore + //The file should have a list of all files to hash (1 filename per line) + //Example: + //.gitignore + // README.md + // view.html + // view.json + $filesToHash = []; + if ($this->localContent->fileExists($moduleLocation . '.fileHashes')) { + $filesToHash = $this->localContent->read($moduleLocation . '.fileHashes'); + + if ($filesToHash && $filesToHash !== '') { + $filesToHash = str_replace("'", '', $filesToHash); + $filesToHash = explode(PHP_EOL, $filesToHash); + } + } + + if ($module['module_type'] === 'views' && $module['is_subview'] == false) { + $moduleLocation = $this->getModuleFilesLocation($module, true); + + $viewFiles = $this->basepackages->utils->scanDir($moduleLocation, true, ['.git/', '.fileHashes']); + + if ($viewFiles && count($viewFiles['files']) > 0) { + $moduleLocationFiles['files'] = array_merge($moduleLocationFiles['files'], $viewFiles['files']); + } + + if ($this->localContent->fileExists($moduleLocation . '.fileHashes')) { + $viewFilesToHash = $this->localContent->read($moduleLocation . '.fileHashes'); + + if ($viewFilesToHash && $viewFilesToHash !== '') { + $viewFilesToHash = str_replace("'", '', $viewFilesToHash); + $viewFilesToHash = explode(PHP_EOL, $viewFilesToHash); + + if (count($viewFilesToHash) > 0) { + $filesToHash = array_merge($filesToHash, $viewFilesToHash); + } + } + } + } + + $filesHash['files_hash'] = []; + + if ($moduleLocationFiles && count($moduleLocationFiles['files']) > 0) { + foreach ($moduleLocationFiles['files'] as $file) { + $filePath = $file; + + $file = str_replace($moduleLocation, '', $file); + + if (count($filesToHash) > 0) { + if (!in_array($file, $filesToHash)) { + continue; + } + } + + $hash = hash_file('md5', base_path($filePath)); + + $filesHash['files_hash'][$file] = $hash; + } + } + + if (isset($filesHash['id'])) { + $this->update($filesHash); + } else { + $this->add($filesHash); + } + } + + return $filesHash; + } + + public function validateFilesHash($module) + { + try { + $moduleLocation = $this->getNewFilesLocation($module); + + $module['repoExists'] = false; + if ($this->localContent->directoryExists($moduleLocation . '.git')) { + $module['repoExists'] = true; + + if (!isset($module['repo_details'])) { + $this->modules->manager->getModuleInfo( + [ + 'module_type' => $module['module_type'], + 'module_id' => $module['id'], + 'sync' => true + ] + ); + } + } + + $filesHash = $this->getFilesHash($module); + + $module['isModified'] = false; + if (!$filesHash) { + $this->reCalculateFilesHash($module, false, false, true); + + return $module; + } + + $moduleFiles = $this->basepackages->utils->scanDir($moduleLocation, true, ['.git/', '.fileHashes']); + + //In case you have .gitignore file, it can get complicated to sort which files to hash. + //So, the files you want to hash, just add them in .fileHashes in the same folder as .gitignore + //The file should have a list of all files to hash (1 filename per line) + //Example: + //.gitignore + // README.md + // view.html + // view.json + $filesToHash = []; + if ($this->localContent->fileExists($moduleLocation . '.fileHashes')) { + $filesToHash = $this->localContent->read($moduleLocation . '.fileHashes'); + + if ($filesToHash && $filesToHash !== '') { + $filesToHash = str_replace("'", '', $filesToHash); + $filesToHash = explode(PHP_EOL, $filesToHash); + } + } + + if ($module['module_type'] === 'views' && $module['is_subview'] == false) { + $moduleLocation = $this->getModuleFilesLocation($module, true); + + $viewFiles = $this->basepackages->utils->scanDir($moduleLocation, true, ['.git/', '.fileHashes']); + + if ($viewFiles && count($viewFiles['files']) > 0) { + $moduleFiles['files'] = array_merge($moduleFiles['files'], $viewFiles['files']); + } + + if ($this->localContent->fileExists($moduleLocation . '.fileHashes')) { + $viewFilesToHash = $this->localContent->read($moduleLocation . '.fileHashes'); + + if ($viewFilesToHash && $viewFilesToHash !== '') { + $viewFilesToHash = str_replace("'", '', $viewFilesToHash); + $viewFilesToHash = explode(PHP_EOL, $viewFilesToHash); + + if (count($viewFilesToHash) > 0) { + $filesToHash = array_merge($filesToHash, $viewFilesToHash); + } + } + } + } + + if ($moduleFiles && count($moduleFiles['files']) > 0) { + foreach ($moduleFiles['files'] as $file) { + $filePath = $file; + + $file = str_replace($moduleLocation, '', $file); + + if (count($filesToHash) > 0) { + if (!in_array($file, $filesToHash)) { + continue; + } + } + + $hash = hash_file('md5', base_path($filePath)); + + if (!isset($filesHash['files_hash'][$file]) || + (isset($filesHash['files_hash'][$file]) && + $filesHash['files_hash'][$file] !== $hash) + ) { + $module['isModified'] = true; + + return $module; + } + } + } + } catch (\throwable | FilesystemException | UnableToCheckExistence $e) { + throw $e; + } + + return $module; + } + public function cleanup($modulePath) { $files = $this->basepackages->utils->scanDir($modulePath); @@ -747,6 +991,8 @@ protected function checkAppType($data) $this->apps->types->add($data); + $this->apps->types->packagesData->last; + $this->addUpdateAppTypeFiles($data); } @@ -1231,7 +1477,7 @@ protected function generateNewFiles(&$data) return true; } - protected function getNewFilesLocation($data, $viewPublic = false) + public function getNewFilesLocation($data, $viewPublic = false) { if ($data['module_type'] === 'components') { $moduleLocation = 'apps/' . ucfirst($data['app_type']) . '/Components/'; @@ -2694,6 +2940,10 @@ public function generateRelease($data) } if (count($releaseData) > 0 && isset($releaseData['newRelease'])) { + if ($module['app_type'] !== 'core' && $module['module_type'] !== 'bundles' && $module['module_type'] !== 'apptypes') { + $this->reCalculateFilesHash($module, false, true); + } + $this->addResponse('Generated New Release!', 0, $releaseData); return true; @@ -3585,7 +3835,8 @@ public function getRemoteModules($id) if (in_array(strtolower($names[1]), $modulesTypeArr)) { $module['module_type'] = $names[1]; } else { - $module['module_type'] = '-'; + unset($modulesArr[$key]); + continue; } } diff --git a/apps/Core/Packages/Devtools/Modules/Install/Schema/FilesHash.php b/apps/Core/Packages/Devtools/Modules/Install/Schema/FilesHash.php new file mode 100644 index 000000000..3703f01fa --- /dev/null +++ b/apps/Core/Packages/Devtools/Modules/Install/Schema/FilesHash.php @@ -0,0 +1,62 @@ + [ + new Column( + 'id', + [ + 'type' => Column::TYPE_INTEGER, + 'notNull' => true, + 'autoIncrement' => true, + 'primary' => true, + ] + ), + new Column( + 'module_type', + [ + 'type' => Column::TYPE_VARCHAR, + 'size' => 50, + 'notNull' => true, + ] + ), + new Column( + 'module_id', + [ + 'type' => Column::TYPE_INTEGER, + 'notNull' => true, + ] + ), + new Column( + 'files_hash', + [ + 'type' => Column::TYPE_JSON, + 'notNull' => true, + ] + ) + ], + 'indexes' => [ + new Index( + 'column_UNIQUE', + [ + 'module_type', + 'module_id' + ], + 'UNIQUE' + ) + ], + 'options' => [ + 'TABLE_COLLATION' => 'utf8mb4_general_ci' + ] + ]; + } +} diff --git a/apps/Core/Packages/Devtools/Modules/Install/package.json b/apps/Core/Packages/Devtools/Modules/Install/package.json index 9786533c1..a27ce65f3 100644 --- a/apps/Core/Packages/Devtools/Modules/Install/package.json +++ b/apps/Core/Packages/Devtools/Modules/Install/package.json @@ -20,9 +20,13 @@ "packages" : [], "middlewares" : [], "views" : [], - "external" : [] + "external" : [], + "externals" : [] }, - "settings" : { - "api_clients" : [1,2] + "settings" : + { + "api_clients" : [1, + 2 + ] } } \ No newline at end of file diff --git a/apps/Core/Packages/Devtools/Modules/Model/AppsCoreDevtoolsFilesHash.php b/apps/Core/Packages/Devtools/Modules/Model/AppsCoreDevtoolsFilesHash.php new file mode 100644 index 000000000..6cb53e74a --- /dev/null +++ b/apps/Core/Packages/Devtools/Modules/Model/AppsCoreDevtoolsFilesHash.php @@ -0,0 +1,16 @@ +helper->decode($package['settings'], true); } - $apiClientServices = $this->basepackages->apiClientServices->getAll(true)->apiClientServices; + $apiClientServices = $this->basepackages->apiClientServices->getApiByAppType(); if (count($package['settings']['api_clients']) > 0) { foreach ($apiClientServices as $api) { diff --git a/apps/Core/Packages/Devtools/Test/DevtoolsTest.php b/apps/Core/Packages/Devtools/Test/DevtoolsTest.php new file mode 100644 index 000000000..2175d45f3 --- /dev/null +++ b/apps/Core/Packages/Devtools/Test/DevtoolsTest.php @@ -0,0 +1,356 @@ +sourceDir))) { + if (!mkdir(base_path($this->sourceDir), 0777, true)) { + return false; + } + } + + //Increase Exectimeout to 3 hours as this process takes time to extract and merge data. + if ((int) ini_get('max_execution_time') < 10800) { + set_time_limit(10800); + } + + //Increase memory_limit to 2G as the process takes a bit of memory to process the array. + if ((int) ini_get('memory_limit') < 1024) { + ini_set('memory_limit', '1024M'); + } + + $this->now = \Carbon\Carbon::now(); + + parent::onConstruct(); + } + + public function __call($method, $arguments) + { + if (method_exists($this, $method)) { + $this->basepackages->progress->updateProgress($method, null, false); + + $call = call_user_func_array([$this, $method], $arguments); + + $callResult = $call; + + if ($call !== false) { + $call = true; + } + + $this->basepackages->progress->updateProgress($method, $call, false); + + return $callResult; + } + } + + protected function testTest() + { + sleep(1); + + return true; + } + + protected function testDownload() + { + $today = $this->now->toDateString(); + + try { + //File is already extracted + if ($this->localContent->fileExists($this->sourceDir . $today . '-funds.db')) { + return true; + } + + //File is already downloaded + if ($this->localContent->fileExists($this->sourceDir . $today . '-funds.db.zst')) { + return true; + } + } catch (FilesystemException | UnableToCheckExistence | UnableToRetrieveMetadata | \throwable $e) { + $this->addResponse($e->getMessage(), 1); + + return false; + } + + $this->method = 'testDownload'; + + return $this->downloadData($this->sourceLink, base_path($this->sourceDir) . $today . '-funds.db.zst'); + } + + protected function downloadData($url, $sink) + { + $download = $this->remoteWebContent->request( + 'GET', + $url, + [ + 'progress' => function( + $downloadTotal, + $downloadedBytes, + $uploadTotal, + $uploadedBytes + ) { + $counters = + [ + 'downloadTotal' => $downloadTotal, + 'downloadedBytes' => $downloadedBytes, + 'uploadTotal' => $uploadTotal, + 'uploadedBytes' => $uploadedBytes + ]; + + if ($downloadedBytes === 0) { + return; + } + + //Trackcounter is needed as guzzelhttp runs this in a while loop causing too many updates with same download count. + //So this way, we only update progress when there is actually an update. + if ($downloadedBytes === $this->trackCounter) { + return; + } + + $this->trackCounter = $downloadedBytes; + + if ($downloadedBytes === $downloadTotal) { + $this->basepackages->progress->updateProgress($this->method, true, false, null, $counters); + } else { + $this->basepackages->progress->updateProgress($this->method, null, false, null, $counters); + } + }, + 'verify' => false, + 'connect_timeout' => 5, + 'sink' => $sink + ] + ); + + $this->trackCounter = 0; + + if ($download->getStatusCode() === 200) { + return true; + } + + $this->addResponse('Download resulted in : ' . $download->getStatusCode(), 1); + + return false; + } + + protected function testProcess() + { + $this->method = 'testProcess'; + + $today = $this->now->toDateString(); + + try { + //File is already extracted + if (!$this->localContent->fileExists($this->sourceDir . $today . '-funds.db')) { + $this->addResponse('File not downloaded and extracted correctly!', 1); + + return false; + } + } catch (FilesystemException | UnableToCheckExistence | \throwable $e) { + $this->addResponse($e->getMessage(), 1); + + return false; + } + + try { + $sqlite = (new Sqlite())->init(base_path($this->sourceDir . $today . '-funds.db')); + } catch (\throwable $e) { + $this->addResponse('Unable to open database file', 1); + + return false; + } + + //Test + // try { + // $this->basepackages->utils->setMicroTimer('DBStart', true); + + // $isinNavs = + // $sqlite->query( + // "SELECT * from nav N + // JOIN securities S ON N.scheme_code = S.scheme_code + // WHERE S.isin = 'INF760K01FV4' + // AND N.date >= '2000-01-01' + // ORDER BY N.date DESC" + // )->fetchAll(Enum::FETCH_ASSOC); + + // $this->basepackages->utils->setMicroTimer('DBStop', true); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: false, dumpTraces: false); + + // $this->basepackages->utils->resetMicroTimer(); + + // $this->basepackages->utils->setMicroTimer('FFReadStart', true); + // $this->navsPackage = new MfNavs; + // $new = $this->navsPackage->getMfNavsByIsin('INF760K01FV4'); + // $this->basepackages->utils->setMicroTimer('FFReadStop', true); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: false, dumpTraces: false); + + // $this->basepackages->utils->resetMicroTimer(); + + // $this->basepackages->utils->setMicroTimer('FFReadStart2', true); + // $this->navsPackage = new MfNavs; + // $new = $this->navsPackage->getById(1); + // $this->basepackages->utils->setMicroTimer('FFReadStop2', true); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: false, dumpTraces: false); + // die(); + // } catch (\throwable $e) { + // trace([$e]); + // } + //Test + + try { + $this->navsPackage = new MfNavs; + + $isins = $sqlite->query("SELECT * from securities")->fetchAll(Enum::FETCH_ASSOC); + + if ($isins && count($isins) > 0) { + $isinsTotal = 1000; + foreach ($isins as $key => $isin) { + $this->basepackages->utils->setMicroTimer('Start'); + $dbIsin = $this->navsPackage->getMfNavsByIsin($isin['isin']); + + // $lastUpdated = $this->now->subDay(1)->toDateString(); + $lastUpdated = '2000-01-01'; + + if (!$dbIsin) { + $dbIsin = []; + $dbIsin['type'] = $isin['type']; + $dbIsin['scheme_code'] = $isin['scheme_code']; + $dbIsin['isin'] = $isin['isin']; + $dbIsin['navs'] = []; + } else { + $lastUpdated = $dbIsin['last_updated']; + } + + $isin = $isin['isin']; + // $this->basepackages->utils->setMicroTimer('DBStart', true); + $isinNavs = + $sqlite->query( + "SELECT * from nav N + JOIN securities S ON N.scheme_code = S.scheme_code + WHERE S.isin = '$isin' + AND N.date >= '$lastUpdated' + ORDER BY N.date ASC" + )->fetchAll(Enum::FETCH_ASSOC); + + // $this->basepackages->utils->setMicroTimer('DBStop', true); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: false, dumpTraces: false); + if ($isinNavs && count($isinNavs) > 0) { + $dbIsin['last_updated'] = $this->helper->last($isinNavs)['date']; + $dbIsin['latest_nav'] = $this->helper->last($isinNavs)['nav']; + foreach ($isinNavs as $isinNav) { + $dbIsin['navs'][$isinNav['date']] = $isinNav['nav']; + } + } + + if (isset($dbIsin['id'])) { + $this->navsPackage->update($dbIsin); + } else { + $this->navsPackage->addMfNavs($dbIsin); + } + // $this->basepackages->utils->setMicroTimer('FFStop', true); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: false, dumpTraces: false); + + // $this->basepackages->utils->setMicroTimer('FFReadStart', true); + // $new = $this->navsPackage->getMfNavsByIsin($isin); + $this->basepackages->utils->setMicroTimer('End'); + + $time = $this->basepackages->utils->getMicroTimer(); + + if ($time && isset($time[1]['difference']) && $time[1]['difference'] !== 0) { + $totalTime = date("H:i:s", floor($time[1]['difference'] * ($isinsTotal - $key))); + // $this->basepackages->utils->formatMicrotime($time[1]['difference'] * $isinsTotal); + } + + $this->basepackages->utils->resetMicroTimer(); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: true, dumpTraces: false); + $this->basepackages->progress->updateProgress( + method: $this->method, + counters: ['stepsTotal' => $isinsTotal, 'stepsCurrent' => ($key + 1)], + text: 'Time remaining : ' . $totalTime . '...' + ); + if ($key === 1000) { + return true; + } + } + } + } catch (\throwable $e) { + trace([$e]); + } + + return true; + // $statement = $sqlite->prepare('SELECT * from nav LIMIT 0,1'); + // trace([$sqlite->query('SELECT * from securities LIMIT 0,10')->fetchAll(Enum::FETCH_ASSOC)]); + // trace([$sqlite->query("SELECT * from nav N JOIN securities S ON N.scheme_code = S.scheme_code WHERE S.isin = 'INF209KB1ZL4' LIMIT 0,10")->fetchAll(Enum::FETCH_ASSOC)]); + // trace([$sqlite->query("SELECT * from nav N JOIN securities S ON N.scheme_code = S.scheme_code WHERE S.isin = 'INF209KB1ZL4'")->fetchAll(Enum::FETCH_ASSOC)]); + } + + // public function getDevtoolsTestById($id) + // { + // $devtoolstest = $this->getById($id); + + // if ($devtoolstest) { + // // + // $this->addResponse('Success'); + + // return; + // } + + // $this->addResponse('Error', 1); + // } + + // public function addDevtoolsTest($data) + // { + // // + // } + + // public function updateDevtoolsTest($data) + // { + // $devtoolstest = $this->getById($id); + + // if ($devtoolstest) { + // // + // $this->addResponse('Success'); + + // return; + // } + + // $this->addResponse('Error', 1); + // } + + // public function removeDevtoolsTest($data) + // { + // $devtoolstest = $this->getById($id); + + // if ($devtoolstest) { + // // + // $this->addResponse('Success'); + + // return; + // } + + // $this->addResponse('Error', 1); + // } +} \ No newline at end of file diff --git a/apps/Core/Packages/Devtools/Test/Install/Schema/DevtoolsTest.php b/apps/Core/Packages/Devtools/Test/Install/Schema/DevtoolsTest.php new file mode 100644 index 000000000..11a1ad91c --- /dev/null +++ b/apps/Core/Packages/Devtools/Test/Install/Schema/DevtoolsTest.php @@ -0,0 +1,69 @@ + [ + new Column( + 'id', + [ + 'type' => Column::TYPE_INTEGER, + 'notNull' => true, + 'autoIncrement' => true, + 'primary' => true, + ] + ), + new Column( + 'first_name', + [ + 'type' => Column::TYPE_VARCHAR, + 'size' => 50, + 'notNull' => true, + ] + ), + new Column( + 'last_name', + [ + 'type' => Column::TYPE_VARCHAR, + 'size' => 50, + 'notNull' => true, + ] + ) + ], + 'indexes' => [ + new Index( + 'column_UNIQUE', + [ + 'last_name' + ], + 'UNIQUE' + ) + ], + 'options' => [ + 'TABLE_COLLATION' => 'utf8mb4_general_ci' + ] + ]; + } + + public function indexes() + { + return + [ + new Index( + 'column_INDEX', + [ + 'first_name' + ], + 'INDEX' + ) + ]; + } +} diff --git a/apps/Core/Packages/Devtools/Test/Install/package.json b/apps/Core/Packages/Devtools/Test/Install/package.json new file mode 100644 index 000000000..ccec11df4 --- /dev/null +++ b/apps/Core/Packages/Devtools/Test/Install/package.json @@ -0,0 +1,39 @@ +{ + "name" : "DevtoolsTest", + "display_name" : "Test", + "description" : "
Test package for devtools
", + "module_type" : "packages", + "app_type" : "core", + "category" : "devtools", + "version" : "0.0.0", + "repo" : "https://.../", + "class" : "Apps\\Core\\Packages\\Devtools\\Test\\DevtoolsTest", + "dependencies" : + { + "core" : + { + "name" : "Core", + "version" : "0.0.0", + "repo" : "https://.../" + }, + "apptype" : [], + "packages" : [], + "middlewares" : [], + "externals" : + { + "composer" : + { + "require" : [] + }, + "config" : + { + "allow-plugins" : [] + }, + "extra" : + { + "patches" : [] + } + } + }, + "settings" : [] +} \ No newline at end of file diff --git a/apps/Core/Packages/Devtools/Test/Model/DevtoolsTest.php b/apps/Core/Packages/Devtools/Test/Model/DevtoolsTest.php new file mode 100644 index 000000000..bb04fadcf --- /dev/null +++ b/apps/Core/Packages/Devtools/Test/Model/DevtoolsTest.php @@ -0,0 +1,14 @@ + + {% set enforce2FADisabled = true %} + {% if core.core['settings']['security']['twofa'] is defined and core.core['settings']['security']['twofa'] == 'true' %} + {% set enforce2FADisabled = false %} + {% endif %}| Name | +Module Type | +Repository | +Modified | +Edit | +
|---|