From de917b289e5d78fe3f34ed039de146aaf0ebc2bb Mon Sep 17 00:00:00 2001 From: "Gurdeep Singh (Guru)" Date: Sun, 23 Feb 2025 23:03:27 +1100 Subject: [PATCH 01/11] fix issue #631 --- system/Base/Installer/View/setup.phtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/Base/Installer/View/setup.phtml b/system/Base/Installer/View/setup.phtml index 05a3218fb..fedf2965c 100644 --- a/system/Base/Installer/View/setup.phtml +++ b/system/Base/Installer/View/setup.phtml @@ -44,7 +44,7 @@
SP Framework Database Update
-
SP Framework Core Setup ()
+
SP Framework Core Setup
From 3d8be48d7ec6a9a350b6574d999cfaf48dc2f6e9 Mon Sep 17 00:00:00 2001 From: "Gurdeep Singh (Guru)" Date: Sun, 23 Feb 2025 23:03:45 +1100 Subject: [PATCH 02/11] fix issue #630 --- system/Base/Providers/DatabaseServiceProvider/Ff/Store.php | 1 + 1 file changed, 1 insertion(+) diff --git a/system/Base/Providers/DatabaseServiceProvider/Ff/Store.php b/system/Base/Providers/DatabaseServiceProvider/Ff/Store.php index fb4a7643e..53420595e 100644 --- a/system/Base/Providers/DatabaseServiceProvider/Ff/Store.php +++ b/system/Base/Providers/DatabaseServiceProvider/Ff/Store.php @@ -133,6 +133,7 @@ public function getSchemaMetaData($relations = false) if (count($relation) > 0 && isset($relation[2])) { try { + $relation[2] = explode('+', $relation[2])[0]; $relationsStore = new Store($relation[2], $this->databasePath, $this->ff); $rmd = array_replace_recursive($rmd, $relationsStore->getSchemaMetaData()); From f109a8023bddb02d340dc3daeca2dd23602249fd Mon Sep 17 00:00:00 2001 From: "Gurdeep Singh (Guru)" Date: Mon, 24 Feb 2025 01:42:45 +1100 Subject: [PATCH 03/11] fix issue #632 --- apps/Core/Components/Register/RegisterComponent.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/Core/Components/Register/RegisterComponent.php b/apps/Core/Components/Register/RegisterComponent.php index 3de58fabf..26678dde7 100644 --- a/apps/Core/Components/Register/RegisterComponent.php +++ b/apps/Core/Components/Register/RegisterComponent.php @@ -114,6 +114,8 @@ public function viewAction() return; } + $this->view->refresh = false; + if (isset($this->getData()['api'])) { $api = $this->api->getById($this->getData()['api']); @@ -144,8 +146,6 @@ public function viewAction() $this->response->setStatusCode(404); return $this->response->send(); - - exit; } $this->view->setLayout('auth'); From bef8819b074e6bf2322572615c95be187b082160 Mon Sep 17 00:00:00 2001 From: "Gurdeep Singh (Guru)" Date: Mon, 24 Feb 2025 03:36:39 +1100 Subject: [PATCH 04/11] fix issue #633 --- .../html/apps/wizard/register/form.html | 4 +++- .../html/apps/wizard/security/form.html | 7 +++++- apps/Core/Views/Default/html/auth/agent.html | 21 ++++++++++++----- apps/Core/Views/Default/html/auth/forgot.html | 21 ++++++++++++----- .../Core/Views/Default/html/auth/pwreset.html | 21 ++++++++++++----- .../Views/Default/html/auth/setup2fa.html | 21 ++++++++++++----- apps/Core/Views/Default/html/auth/view.html | 23 ++++++++++++++----- .../AccessServiceProvider/Access/Auth.php | 21 ++++++++++++++--- .../Providers/MiddlewaresServiceProvider.php | 4 ++-- 9 files changed, 106 insertions(+), 37 deletions(-) diff --git a/apps/Core/Views/Default/html/apps/wizard/register/form.html b/apps/Core/Views/Default/html/apps/wizard/register/form.html index 3fda3bf97..544431f63 100644 --- a/apps/Core/Views/Default/html/apps/wizard/register/form.html +++ b/apps/Core/Views/Default/html/apps/wizard/register/form.html @@ -33,7 +33,9 @@ {% endif %} {% set enforce2FA = false %} {% if app['enforce_2fa'] == true %} - {% set enforce2FA = true %} + {% if core.core['settings']['security']['twofa'] is defined and core.core['settings']['security']['twofa'] == 'true' %} + {% set enforce2FA = true %} + {% endif %} {% endif %} {% set appRegistrationRoleId = app['registration_role_id'] %} diff --git a/apps/Core/Views/Default/html/apps/wizard/security/form.html b/apps/Core/Views/Default/html/apps/wizard/security/form.html index 061c384b2..6d361df98 100644 --- a/apps/Core/Views/Default/html/apps/wizard/security/form.html +++ b/apps/Core/Views/Default/html/apps/wizard/security/form.html @@ -80,6 +80,10 @@ ] )}}
+ {% set enforce2FADisabled = true %} + {% if core.core['settings']['security']['twofa'] is defined and core.core['settings']['security']['twofa'] == 'true' %} + {% set enforce2FADisabled = false %} + {% endif %}
{{adminltetags.useTag('fields', [ @@ -91,7 +95,8 @@ 'fieldLabel' : 'Enforce 2FA at login?', 'fieldType' : 'checkbox', 'fieldHelp' : true, - 'fieldHelpTooltipContent' : 'Enforce user accounts to login/enable 2FA.', + 'fieldDisabled' : enforce2FADisabled, + 'fieldHelpTooltipContent' : 'Enforce user accounts to login/enable 2FA. Will only work if twofa is enabled in core security settings.', 'fieldCheckboxType' : 'success', 'fieldCheckboxChecked' : enforce2FA, 'fieldCheckboxAdditionClass' : 'text-sm text-uppercase', diff --git a/apps/Core/Views/Default/html/auth/agent.html b/apps/Core/Views/Default/html/auth/agent.html index 140ee718b..5a8eb6fa7 100644 --- a/apps/Core/Views/Default/html/auth/agent.html +++ b/apps/Core/Views/Default/html/auth/agent.html @@ -200,6 +200,12 @@ return (className.match (/(^|\s)fa-\S+/g) || []).join(' '); }).addClass('fa-exclamation-circle'); } + } else if ($('#{{componentId}}-form-generate-code').length > 0) { + url = '{{links.url("auth/enableTwoFaOtp")}}'; + postData = { + 'user' : $('#{{componentId}}-form-user').val().trim(), + 'pass' : $('#{{componentId}}-form-pass').val().trim() + }; } else if ($('#{{componentId}}-form-verify').length > 0) { if (thisButton.id === '{{componentId}}-form-agent-verification-email_code') { url = '{{links.url("auth/sendVerification")}}'; @@ -209,12 +215,6 @@ }; url = '{{links.url("auth/verify")}}'; } - } else if ($('#{{componentId}}-form-generate-code').length > 0) { - url = '{{links.url("auth/enableTwoFaOtp")}}'; - postData = { - 'user' : $('#{{componentId}}-form-user').val().trim(), - 'pass' : $('#{{componentId}}-form-pass').val().trim() - }; } postData[$('#security-token').attr('name')] = $('#security-token').val(); } @@ -403,10 +403,17 @@ $('#{{componentId}}-form-twofa-otp').parents('.form-group').removeClass('d-none'); $('#twofa-otp').removeClass('d-none'); $('#{{componentId}}-form-twofa-otp').focus(); + if (response.responseData['allowed_methods'].includes('email')) { + $('#{{componentId}}-form-twofa-otp-switch_to_email').removeClass('d-none') + } } else if (response.responseData['allowed_methods'].includes('email')) { $('#{{componentId}}-form :input').prop('disabled', false); $('#{{componentId}}-form-twofa-email').parents('.form-group').removeClass('d-none'); $('#twofa-email').removeClass('d-none'); + $('#{{componentId}}-form-twofa-email').focus(); + if (response.responseData['allowed_methods'].includes('otp')) { + $('#{{componentId}}-form-twofa-otp-switch_to_otp').removeClass('d-none') + } } setTimeout(function () { @@ -736,11 +743,13 @@ $('#{{componentId}}-form-twofa-otp').val(''); $('#twofa-otp').addClass('d-none'); $('#twofa-email').removeClass('d-none'); + $('#{{componentId}}-form-twofa-email-switch_to_otp').removeClass('d-none') }); $('#{{componentId}}-form-twofa-email-switch_to_otp').click(function() { $('#{{componentId}}-form-twofa-email').val(''); $('#twofa-otp').removeClass('d-none'); $('#twofa-email').addClass('d-none'); + $('#{{componentId}}-form-twofa-otp-switch_to_email').removeClass('d-none') }); }); \ No newline at end of file diff --git a/apps/Core/Views/Default/html/auth/forgot.html b/apps/Core/Views/Default/html/auth/forgot.html index a432212e6..6f1e5ccd4 100644 --- a/apps/Core/Views/Default/html/auth/forgot.html +++ b/apps/Core/Views/Default/html/auth/forgot.html @@ -184,6 +184,12 @@ return (className.match (/(^|\s)fa-\S+/g) || []).join(' '); }).addClass('fa-exclamation-circle'); } + } else if ($('#{{componentId}}-form-generate-code').length > 0) { + url = '{{links.url("auth/enableTwoFaOtp")}}'; + postData = { + 'user' : $('#{{componentId}}-form-user').val().trim(), + 'pass' : $('#{{componentId}}-form-pass').val().trim() + }; } else if ($('#{{componentId}}-form-verify').length > 0) { if (thisButton.id === '{{componentId}}-form-agent-verification-email_code') { url = '{{links.url("auth/sendVerification")}}'; @@ -193,12 +199,6 @@ }; url = '{{links.url("auth/verify")}}'; } - } else if ($('#{{componentId}}-form-generate-code').length > 0) { - url = '{{links.url("auth/enableTwoFaOtp")}}'; - postData = { - 'user' : $('#{{componentId}}-form-user').val().trim(), - 'pass' : $('#{{componentId}}-form-pass').val().trim() - }; } postData[$('#security-token').attr('name')] = $('#security-token').val(); } @@ -387,10 +387,17 @@ $('#{{componentId}}-form-twofa-otp').parents('.form-group').removeClass('d-none'); $('#twofa-otp').removeClass('d-none'); $('#{{componentId}}-form-twofa-otp').focus(); + if (response.responseData['allowed_methods'].includes('email')) { + $('#{{componentId}}-form-twofa-otp-switch_to_email').removeClass('d-none') + } } else if (response.responseData['allowed_methods'].includes('email')) { $('#{{componentId}}-form :input').prop('disabled', false); $('#{{componentId}}-form-twofa-email').parents('.form-group').removeClass('d-none'); $('#twofa-email').removeClass('d-none'); + $('#{{componentId}}-form-twofa-email').focus(); + if (response.responseData['allowed_methods'].includes('otp')) { + $('#{{componentId}}-form-twofa-otp-switch_to_otp').removeClass('d-none') + } } setTimeout(function () { @@ -720,11 +727,13 @@ $('#{{componentId}}-form-twofa-otp').val(''); $('#twofa-otp').addClass('d-none'); $('#twofa-email').removeClass('d-none'); + $('#{{componentId}}-form-twofa-email-switch_to_otp').removeClass('d-none') }); $('#{{componentId}}-form-twofa-email-switch_to_otp').click(function() { $('#{{componentId}}-form-twofa-email').val(''); $('#twofa-otp').removeClass('d-none'); $('#twofa-email').addClass('d-none'); + $('#{{componentId}}-form-twofa-otp-switch_to_email').removeClass('d-none') }); }); \ No newline at end of file diff --git a/apps/Core/Views/Default/html/auth/pwreset.html b/apps/Core/Views/Default/html/auth/pwreset.html index 0cfb496de..d3b2454c4 100644 --- a/apps/Core/Views/Default/html/auth/pwreset.html +++ b/apps/Core/Views/Default/html/auth/pwreset.html @@ -225,6 +225,12 @@ return (className.match (/(^|\s)fa-\S+/g) || []).join(' '); }).addClass('fa-exclamation-circle'); } + } else if ($('#{{componentId}}-form-generate-code').length > 0) { + url = '{{links.url("auth/enableTwoFaOtp")}}'; + postData = { + 'user' : $('#{{componentId}}-form-user').val().trim(), + 'pass' : $('#{{componentId}}-form-pass').val().trim() + }; } else if ($('#{{componentId}}-form-verify').length > 0) { if (thisButton.id === '{{componentId}}-form-agent-verification-email_code') { url = '{{links.url("auth/sendVerification")}}'; @@ -234,12 +240,6 @@ }; url = '{{links.url("auth/verify")}}'; } - } else if ($('#{{componentId}}-form-generate-code').length > 0) { - url = '{{links.url("auth/enableTwoFaOtp")}}'; - postData = { - 'user' : $('#{{componentId}}-form-user').val().trim(), - 'pass' : $('#{{componentId}}-form-pass').val().trim() - }; } postData[$('#security-token').attr('name')] = $('#security-token').val(); } @@ -428,10 +428,17 @@ $('#{{componentId}}-form-twofa-otp').parents('.form-group').removeClass('d-none'); $('#twofa-otp').removeClass('d-none'); $('#{{componentId}}-form-twofa-otp').focus(); + if (response.responseData['allowed_methods'].includes('email')) { + $('#{{componentId}}-form-twofa-otp-switch_to_email').removeClass('d-none') + } } else if (response.responseData['allowed_methods'].includes('email')) { $('#{{componentId}}-form :input').prop('disabled', false); $('#{{componentId}}-form-twofa-email').parents('.form-group').removeClass('d-none'); $('#twofa-email').removeClass('d-none'); + $('#{{componentId}}-form-twofa-email').focus(); + if (response.responseData['allowed_methods'].includes('otp')) { + $('#{{componentId}}-form-twofa-otp-switch_to_otp').removeClass('d-none') + } } setTimeout(function () { @@ -761,11 +768,13 @@ $('#{{componentId}}-form-twofa-otp').val(''); $('#twofa-otp').addClass('d-none'); $('#twofa-email').removeClass('d-none'); + $('#{{componentId}}-form-twofa-email-switch_to_otp').removeClass('d-none') }); $('#{{componentId}}-form-twofa-email-switch_to_otp').click(function() { $('#{{componentId}}-form-twofa-email').val(''); $('#twofa-otp').removeClass('d-none'); $('#twofa-email').addClass('d-none'); + $('#{{componentId}}-form-twofa-otp-switch_to_email').removeClass('d-none') }); }); \ No newline at end of file diff --git a/apps/Core/Views/Default/html/auth/setup2fa.html b/apps/Core/Views/Default/html/auth/setup2fa.html index fe0a72dd7..aab4d7f1f 100644 --- a/apps/Core/Views/Default/html/auth/setup2fa.html +++ b/apps/Core/Views/Default/html/auth/setup2fa.html @@ -264,6 +264,12 @@ return (className.match (/(^|\s)fa-\S+/g) || []).join(' '); }).addClass('fa-exclamation-circle'); } + } else if ($('#{{componentId}}-form-generate-code').length > 0) { + url = '{{links.url("auth/enableTwoFaOtp")}}'; + postData = { + 'user' : $('#{{componentId}}-form-user').val().trim(), + 'pass' : $('#{{componentId}}-form-pass').val().trim() + }; } else if ($('#{{componentId}}-form-verify').length > 0) { if (thisButton.id === '{{componentId}}-form-agent-verification-email_code') { url = '{{links.url("auth/sendVerification")}}'; @@ -273,12 +279,6 @@ }; url = '{{links.url("auth/verify")}}'; } - } else if ($('#{{componentId}}-form-generate-code').length > 0) { - url = '{{links.url("auth/enableTwoFaOtp")}}'; - postData = { - 'user' : $('#{{componentId}}-form-user').val().trim(), - 'pass' : $('#{{componentId}}-form-pass').val().trim() - }; } postData[$('#security-token').attr('name')] = $('#security-token').val(); } @@ -467,10 +467,17 @@ $('#{{componentId}}-form-twofa-otp').parents('.form-group').removeClass('d-none'); $('#twofa-otp').removeClass('d-none'); $('#{{componentId}}-form-twofa-otp').focus(); + if (response.responseData['allowed_methods'].includes('email')) { + $('#{{componentId}}-form-twofa-otp-switch_to_email').removeClass('d-none') + } } else if (response.responseData['allowed_methods'].includes('email')) { $('#{{componentId}}-form :input').prop('disabled', false); $('#{{componentId}}-form-twofa-email').parents('.form-group').removeClass('d-none'); $('#twofa-email').removeClass('d-none'); + $('#{{componentId}}-form-twofa-email').focus(); + if (response.responseData['allowed_methods'].includes('otp')) { + $('#{{componentId}}-form-twofa-otp-switch_to_otp').removeClass('d-none') + } } setTimeout(function () { @@ -800,11 +807,13 @@ $('#{{componentId}}-form-twofa-otp').val(''); $('#twofa-otp').addClass('d-none'); $('#twofa-email').removeClass('d-none'); + $('#{{componentId}}-form-twofa-email-switch_to_otp').removeClass('d-none') }); $('#{{componentId}}-form-twofa-email-switch_to_otp').click(function() { $('#{{componentId}}-form-twofa-email').val(''); $('#twofa-otp').removeClass('d-none'); $('#twofa-email').addClass('d-none'); + $('#{{componentId}}-form-twofa-otp-switch_to_email').removeClass('d-none') }); }); \ No newline at end of file diff --git a/apps/Core/Views/Default/html/auth/view.html b/apps/Core/Views/Default/html/auth/view.html index 0d82f883f..41c291af9 100644 --- a/apps/Core/Views/Default/html/auth/view.html +++ b/apps/Core/Views/Default/html/auth/view.html @@ -65,6 +65,7 @@ 'fieldPlaceholder' : '2FA OTP Code', 'fieldGroupPreAddonButtonId' : 'switch_to_email', 'fieldGroupPreAddonButtonValue' : '', + 'fieldGroupPreAddonButtonHidden' : true, 'fieldGroupPreAddonButtonTooltipTitle' : 'Switch to 2FA (Email)', 'fieldDataInputMinLength' : 4, 'fieldDataInputMaxLength' : 8 @@ -89,6 +90,7 @@ 'fieldGroupPostAddonButtonValue' : '', 'fieldGroupPostAddonButtonTooltipTitle' : 'Email new Code', 'fieldGroupPreAddonButtonId' : 'switch_to_otp', + 'fieldGroupPreAddonButtonHidden' : true, 'fieldGroupPreAddonButtonValue' : '', 'fieldGroupPreAddonButtonTooltipTitle' : 'Switch to 2FA (OTP)', 'fieldDataInputMinLength' : 4, @@ -280,6 +282,12 @@ return (className.match (/(^|\s)fa-\S+/g) || []).join(' '); }).addClass('fa-exclamation-circle'); } + } else if ($('#{{componentId}}-form-generate-code').length > 0) { + url = '{{links.url("auth/enableTwoFaOtp")}}'; + postData = { + 'user' : $('#{{componentId}}-form-user').val().trim(), + 'pass' : $('#{{componentId}}-form-pass').val().trim() + }; } else if ($('#{{componentId}}-form-verify').length > 0) { if (thisButton.id === '{{componentId}}-form-agent-verification-email_code') { url = '{{links.url("auth/sendVerification")}}'; @@ -289,12 +297,6 @@ }; url = '{{links.url("auth/verify")}}'; } - } else if ($('#{{componentId}}-form-generate-code').length > 0) { - url = '{{links.url("auth/enableTwoFaOtp")}}'; - postData = { - 'user' : $('#{{componentId}}-form-user').val().trim(), - 'pass' : $('#{{componentId}}-form-pass').val().trim() - }; } postData[$('#security-token').attr('name')] = $('#security-token').val(); } @@ -483,10 +485,17 @@ $('#{{componentId}}-form-twofa-otp').parents('.form-group').removeClass('d-none'); $('#twofa-otp').removeClass('d-none'); $('#{{componentId}}-form-twofa-otp').focus(); + if (response.responseData['allowed_methods'].includes('email')) { + $('#{{componentId}}-form-twofa-otp-switch_to_email').removeClass('d-none') + } } else if (response.responseData['allowed_methods'].includes('email')) { $('#{{componentId}}-form :input').prop('disabled', false); $('#{{componentId}}-form-twofa-email').parents('.form-group').removeClass('d-none'); $('#twofa-email').removeClass('d-none'); + $('#{{componentId}}-form-twofa-email').focus(); + if (response.responseData['allowed_methods'].includes('otp')) { + $('#{{componentId}}-form-twofa-otp-switch_to_otp').removeClass('d-none') + } } setTimeout(function () { @@ -816,11 +825,13 @@ $('#{{componentId}}-form-twofa-otp').val(''); $('#twofa-otp').addClass('d-none'); $('#twofa-email').removeClass('d-none'); + $('#{{componentId}}-form-twofa-email-switch_to_otp').removeClass('d-none') }); $('#{{componentId}}-form-twofa-email-switch_to_otp').click(function() { $('#{{componentId}}-form-twofa-email').val(''); $('#twofa-otp').removeClass('d-none'); $('#twofa-email').addClass('d-none'); + $('#{{componentId}}-form-twofa-otp-switch_to_email').removeClass('d-none') }); }); \ No newline at end of file diff --git a/system/Base/Providers/AccessServiceProvider/Access/Auth.php b/system/Base/Providers/AccessServiceProvider/Access/Auth.php index e60021fca..d0fb3763b 100644 --- a/system/Base/Providers/AccessServiceProvider/Access/Auth.php +++ b/system/Base/Providers/AccessServiceProvider/Access/Auth.php @@ -82,7 +82,16 @@ public function attempt($data) if (str_contains(strtolower($validate), 'please contact administrator')) { $validate = str_replace('Error! Please contact administrator.', '', $validate); } - $this->addResponse($validate, 3, ['allowed_methods' => $this->core->core['settings']['security']['twofaSettings']['twofaUsing']]); + + if ($this->account['security']['twofa_otp_status'] == true) { + $this->addResponse($validate, 3, ['allowed_methods' => $this->core->core['settings']['security']['twofaSettings']['twofaUsing']]); + } else {//redirect to setup twofa + $this->addResponse('2FA needed, but not set. Redirecting...'); + + $this->packagesData->redirectUrl = $this->links->url('auth/q/setup2fa/true'); + + return true; + } } else { $this->addResponse($validate, 1); } @@ -94,7 +103,10 @@ public function attempt($data) $security = $this->getAccountSecurityObject(); - if (isset($this->app['enforce_2fa']) && $this->app['enforce_2fa'] == '1') { + if ($this->core->core['settings']['security']['twofa'] == 'true' && + isset($this->app['enforce_2fa']) && + $this->app['enforce_2fa'] == '1' + ) { if (!$this->twoFa->validateTwoFaCode($security, $data)) { $this->addResponse( $this->twoFa->packagesData->responseMessage, @@ -749,7 +761,10 @@ public function validateData(array $data, $task) $this->validation->add('user', PresenceOf::class, ["message" => "Enter valid user name."]); $this->validation->add('pass', PresenceOf::class, ["message" => "Enter valid password."]); if ($task === 'auth2fa') { - if (isset($this->app['enforce_2fa']) && $this->app['enforce_2fa'] == '1') { + if ($this->core->core['settings']['security']['twofa'] == 'true' && + isset($this->app['enforce_2fa']) && + $this->app['enforce_2fa'] == '1' + ) { $this->validation->add('twofa_using', PresenceOf::class, ["message" => "Error! Please contact administrator."]); $this->validation->add('code', PresenceOf::class, ["message" => "Enter valid 2FA code"]); if (isset($data['twofa_using'])) { diff --git a/system/Base/Providers/MiddlewaresServiceProvider.php b/system/Base/Providers/MiddlewaresServiceProvider.php index e271d62c4..ec2bef74e 100644 --- a/system/Base/Providers/MiddlewaresServiceProvider.php +++ b/system/Base/Providers/MiddlewaresServiceProvider.php @@ -158,8 +158,8 @@ protected function checkRoute($middleware) $this->data['appRoute'] . '/auth/pwreset', $this->data['appRoute'] . '/auth/checkpwstrength', $this->data['appRoute'] . '/auth/generatepw', - $this->data['appRoute'] . '/auth/enabletwofatotp', - $this->data['appRoute'] . '/auth/verifytwofatotp', + $this->data['appRoute'] . '/auth/enabletwofaotp', + $this->data['appRoute'] . '/auth/verifytwofaotp', $this->data['appRoute'] . '/auth/logout', $this->data['appRoute'] . '/auth/sendverification', $this->data['appRoute'] . '/auth/verify', From f7e73e85a439cfe507527d4044a49bb316cad142 Mon Sep 17 00:00:00 2001 From: "Gurdeep Singh (Guru)" Date: Mon, 24 Feb 2025 13:50:25 +1100 Subject: [PATCH 05/11] fix issue #635. Also fix bug with modules and devtools settings modifying app_type of all repos. --- .../Api/Client/Services/ServicesComponent.php | 5 +++ .../Content/Listing/Table/DynamicTable.php | 38 +++++++++++++++++-- .../Adminltetags/Traits/DynamicTable.php | 4 ++ .../Packages/Devtools/Modules/Settings.php | 2 +- .../api/client/services/apis/common.html | 2 +- .../services/apis/repos/gitea/form.html | 10 ++++- .../services/apis/repos/github/form.html | 10 ++++- .../services/apis/repos/github/view.html | 2 +- .../html/system/api/client/services/view.html | 4 ++ .../ModulesServiceProvider/Settings.php | 2 +- 10 files changed, 67 insertions(+), 12 deletions(-) diff --git a/apps/Core/Components/System/Api/Client/Services/ServicesComponent.php b/apps/Core/Components/System/Api/Client/Services/ServicesComponent.php index 8c161be78..fe096f34f 100644 --- a/apps/Core/Components/System/Api/Client/Services/ServicesComponent.php +++ b/apps/Core/Components/System/Api/Client/Services/ServicesComponent.php @@ -27,6 +27,10 @@ public function viewAction() $this->view->apiLocations = $this->apiPackage->apiLocations; if (isset($this->getData()['id'])) { + if (isset($this->getData()['clone'])) { + $this->view->clone = true; + } + if ($this->getData()['id'] != 0) { $api = $this->apiPackage->getApiById($this->getData()['id']); @@ -96,6 +100,7 @@ function ($dataArr) { 'actionsToEnable' => [ 'edit' => 'system/api/client/services', + 'clone' => 'system/api/client/services', 'remove' => 'system/api/client/services/remove' ] ]; diff --git a/apps/Core/Packages/Adminltetags/Tags/Content/Listing/Table/DynamicTable.php b/apps/Core/Packages/Adminltetags/Tags/Content/Listing/Table/DynamicTable.php index 621ea5a61..e652bd56e 100644 --- a/apps/Core/Packages/Adminltetags/Tags/Content/Listing/Table/DynamicTable.php +++ b/apps/Core/Packages/Adminltetags/Tags/Content/Listing/Table/DynamicTable.php @@ -368,14 +368,12 @@ protected function generateRowsContent() } foreach ($column as $controlKey => $control) { - $this->dtParams['dtControlsLinkClass'] = isset($this->params['dtControlsLinkClass']) ? $this->params['dtControlsLinkClass'] : 'contentAjaxLink'; if ($controlKey === 'view') { - if (is_array($control)) { $control = $control['link']; $title = @@ -410,7 +408,6 @@ protected function generateRowsContent() ] ); } else if ($controlKey === 'edit') { - if (is_array($control)) { $control = $control['link']; $title = @@ -444,8 +441,41 @@ protected function generateRowsContent() ] ] ); - } else if ($controlKey === 'remove') { + } else if ($controlKey === 'clone') { + if (is_array($control)) { + $control = $control['link']; + $title = + isset($control['title']) ? + strtoupper($control['title']) : + 'CLONE'; + $icon = + isset($control['icon']) ? + strtoupper($control['icon']) : + 'clone'; + $type = + isset($control['type']) ? + strtoupper($control['type']) : + 'primary'; + } else { + $title = 'CLONE'; + $icon = 'clone'; + $type = 'primary'; + } + + $controlButtons = array_merge($controlButtons, + [ + $controlKey => + [ + 'title' => $title, + 'additionalClass' => 'rowEdit ' . $this->dtParams['dtControlsLinkClass'], + 'icon' => $icon, + 'buttonType' => $type, + 'link' => $control + ] + ] + ); + } else if ($controlKey === 'remove') { if (is_array($control)) { $control = $control['link']; $title = diff --git a/apps/Core/Packages/Adminltetags/Traits/DynamicTable.php b/apps/Core/Packages/Adminltetags/Traits/DynamicTable.php index bb9c8488e..b004cf455 100644 --- a/apps/Core/Packages/Adminltetags/Traits/DynamicTable.php +++ b/apps/Core/Packages/Adminltetags/Traits/DynamicTable.php @@ -219,6 +219,10 @@ public function generateDTContent( $actions[$key] = $this->links->url($action . '/q/id/' . $row['id']); } } + + if ($key === 'clone') { + $actions[$key] = $actions[$key] . '/clone/true'; + } } $row["__control"] = $actions; diff --git a/apps/Core/Packages/Devtools/Modules/Settings.php b/apps/Core/Packages/Devtools/Modules/Settings.php index 727d40541..b7dfeff61 100644 --- a/apps/Core/Packages/Devtools/Modules/Settings.php +++ b/apps/Core/Packages/Devtools/Modules/Settings.php @@ -12,7 +12,7 @@ public function afterUpdate($packageClass, $package, $data) $package['settings'] = $this->helper->decode($package['settings'], true); } - $apiClientServices = $this->basepackages->apiClientServices->getAll(true)->apiClientServices; + $apiClientServices = $this->basepackages->apiClientServices->getApiByAppType(); if (count($package['settings']['api_clients']) > 0) { foreach ($apiClientServices as $api) { diff --git a/apps/Core/Views/Default/html/system/api/client/services/apis/common.html b/apps/Core/Views/Default/html/system/api/client/services/apis/common.html index 0de23e02c..be29fb9e2 100644 --- a/apps/Core/Views/Default/html/system/api/client/services/apis/common.html +++ b/apps/Core/Views/Default/html/system/api/client/services/apis/common.html @@ -165,7 +165,7 @@
{% if apiInUse == 1 %} {% set inUse = '
Yes
' %} - {% elseif apiInUse == 0 %} + {% elseif apiInUse == 0 or apiInUse == '' %} {% set inUse = '
No
' %} {% endif %} {{adminltetags.useTag('fields', diff --git a/apps/Core/Views/Default/html/system/api/client/services/apis/repos/gitea/form.html b/apps/Core/Views/Default/html/system/api/client/services/apis/repos/gitea/form.html index b42603d05..114b06e82 100644 --- a/apps/Core/Views/Default/html/system/api/client/services/apis/repos/gitea/form.html +++ b/apps/Core/Views/Default/html/system/api/client/services/apis/repos/gitea/form.html @@ -1,11 +1,17 @@ {% set locationDisabled = false %} {% if api['id'] is defined %} {% set apiId = api['id'] %} - {% set apiCategoryId = api['api_category_id'] %} {% set apiName = api['name'] %} - {% set apiProvider = api['provider']|lower %} {% set apiInUse = api['in_use'] %} {% set apiUsedBy = api['used_by'] %} + {% if clone is defined and clone == true %} + {% set apiId = '' %} + {% set apiName = api['name'] ~ ' (Clone)' %} + {% set apiInUse = '' %} + {% set apiUsedBy = '' %} + {% endif %} + {% set apiCategoryId = api['api_category_id'] %} + {% set apiProvider = api['provider']|lower %} {% set apiSetup = api['setup'] %} {% set apiLocation = api['location'] %} {% set apiDescription = api['description'] %} diff --git a/apps/Core/Views/Default/html/system/api/client/services/apis/repos/github/form.html b/apps/Core/Views/Default/html/system/api/client/services/apis/repos/github/form.html index 0471868b2..bf8de1c7f 100644 --- a/apps/Core/Views/Default/html/system/api/client/services/apis/repos/github/form.html +++ b/apps/Core/Views/Default/html/system/api/client/services/apis/repos/github/form.html @@ -1,11 +1,17 @@ {% set locationDisabled = false %} {% if api['id'] is defined %} {% set apiId = api['id'] %} - {% set apiCategoryId = api['api_category_id'] %} {% set apiName = api['name'] %} - {% set apiProvider = api['provider']|lower %} {% set apiInUse = api['in_use'] %} {% set apiUsedBy = api['used_by'] %} + {% if clone is defined and clone == true %} + {% set apiId = '' %} + {% set apiName = api['name'] ~ ' (Clone)' %} + {% set apiInUse = '' %} + {% set apiUsedBy = '' %} + {% endif %} + {% set apiCategoryId = api['api_category_id'] %} + {% set apiProvider = api['provider']|lower %} {% set apiSetup = api['setup'] %} {% set apiLocation = api['location'] %} {% set apiDescription = api['description'] %} diff --git a/apps/Core/Views/Default/html/system/api/client/services/apis/repos/github/view.html b/apps/Core/Views/Default/html/system/api/client/services/apis/repos/github/view.html index c4986485e..d56f6d45e 100644 --- a/apps/Core/Views/Default/html/system/api/client/services/apis/repos/github/view.html +++ b/apps/Core/Views/Default/html/system/api/client/services/apis/repos/github/view.html @@ -12,7 +12,7 @@ 'cardIcon' : 'exchange-alt', 'cardTitle' : title, 'cardAdditionalClass' : 'rounded-0', - 'cardBodyInclude' : 'services/apis/repos/gitea/form', + 'cardBodyInclude' : 'services/apis/repos/github/form', 'formButtons' : [ 'updateButtonId' : apiId, diff --git a/apps/Core/Views/Default/html/system/api/client/services/view.html b/apps/Core/Views/Default/html/system/api/client/services/view.html index 4509c50a1..59ff5869c 100644 --- a/apps/Core/Views/Default/html/system/api/client/services/view.html +++ b/apps/Core/Views/Default/html/system/api/client/services/view.html @@ -1,6 +1,10 @@ {% if api['id'] is defined %} {% set apiId = api['id'] %} {% set title = 'API:' ~ api['category'] ~ ':' ~ api['provider'] ~ ' - ' ~ api['name'] %} + {% if clone is defined and clone == true %} + {% set apiId = '' %} + {% set title = title ~ ' (Clone)' %} + {% endif %} {% else %} {% set apiId = '' %} {% set title = 'New API:' ~ api['category'] ~ ':' ~ api['provider'] %} diff --git a/system/Base/Providers/ModulesServiceProvider/Settings.php b/system/Base/Providers/ModulesServiceProvider/Settings.php index d442eb7a6..b263a0b36 100644 --- a/system/Base/Providers/ModulesServiceProvider/Settings.php +++ b/system/Base/Providers/ModulesServiceProvider/Settings.php @@ -25,7 +25,7 @@ public function afterUpdate($packageClass, $package, $data) $package['settings'] = $this->helper->decode($package['settings'], true); } - $apiClientServices = $this->basepackages->apiClientServices->getAll(true)->apiClientServices; + $apiClientServices = $this->basepackages->apiClientServices->getApiByAppType(); if (count($package['settings']['api_clients']) > 0) { foreach ($apiClientServices as $api) { From a31a46b08aa9afd9fa067020ef90f550757ca1a1 Mon Sep 17 00:00:00 2001 From: "Gurdeep Singh (Guru)" Date: Tue, 25 Feb 2025 06:37:26 +1100 Subject: [PATCH 06/11] fix issue #636 --- .../Views/Default/html/devtools/modules/module/info.html | 6 +++--- .../Installer/Packages/Setup/Schema/Modules/Bundles.php | 2 +- .../Installer/Packages/Setup/Schema/Modules/Components.php | 6 +++--- .../Installer/Packages/Setup/Schema/Modules/Externals.php | 4 ++-- .../Installer/Packages/Setup/Schema/Modules/Middlewares.php | 6 +++--- .../Installer/Packages/Setup/Schema/Modules/Packages.php | 6 +++--- .../Base/Installer/Packages/Setup/Schema/Modules/Views.php | 6 +++--- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/apps/Core/Views/Default/html/devtools/modules/module/info.html b/apps/Core/Views/Default/html/devtools/modules/module/info.html index 53b36e522..bde5b1ffc 100644 --- a/apps/Core/Views/Default/html/devtools/modules/module/info.html +++ b/apps/Core/Views/Default/html/devtools/modules/module/info.html @@ -240,7 +240,7 @@ 'fieldBazPostOnUpdate' : true, 'fieldBazPostOnUpdate' : true, 'fieldDataInputMinLength' : 1, - 'fieldDataInputMaxLength' : 50, + 'fieldDataInputMaxLength' : 100, 'fieldValue' : moduleName ] )}} @@ -264,7 +264,7 @@ 'fieldBazPostOnUpdate' : true, 'fieldBazPostOnUpdate' : true, 'fieldDataInputMinLength' : 1, - 'fieldDataInputMaxLength' : 50, + 'fieldDataInputMaxLength' : 100, 'fieldValue' : moduleRoute ] )}} @@ -289,7 +289,7 @@ 'fieldBazScan' : true, 'fieldBazPostOnUpdate' : true, 'fieldDataInputMinLength' : 1, - 'fieldDataInputMaxLength' : 50, + 'fieldDataInputMaxLength' : 255, 'fieldValue' : moduleDisplay_name ] )}} diff --git a/system/Base/Installer/Packages/Setup/Schema/Modules/Bundles.php b/system/Base/Installer/Packages/Setup/Schema/Modules/Bundles.php index e015ad5d4..79e1aaade 100644 --- a/system/Base/Installer/Packages/Setup/Schema/Modules/Bundles.php +++ b/system/Base/Installer/Packages/Setup/Schema/Modules/Bundles.php @@ -25,7 +25,7 @@ public function columns() 'name', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 100, 'notNull' => true, ] ), diff --git a/system/Base/Installer/Packages/Setup/Schema/Modules/Components.php b/system/Base/Installer/Packages/Setup/Schema/Modules/Components.php index 6f891b8b0..3ffd09f43 100644 --- a/system/Base/Installer/Packages/Setup/Schema/Modules/Components.php +++ b/system/Base/Installer/Packages/Setup/Schema/Modules/Components.php @@ -25,7 +25,7 @@ public function columns() 'name', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 100, 'notNull' => true, ] ), @@ -33,7 +33,7 @@ public function columns() 'route', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 100, 'notNull' => true, ] ), @@ -65,7 +65,7 @@ public function columns() 'category', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 100, 'notNull' => true, ] ), diff --git a/system/Base/Installer/Packages/Setup/Schema/Modules/Externals.php b/system/Base/Installer/Packages/Setup/Schema/Modules/Externals.php index 047432e72..df1756c5d 100644 --- a/system/Base/Installer/Packages/Setup/Schema/Modules/Externals.php +++ b/system/Base/Installer/Packages/Setup/Schema/Modules/Externals.php @@ -33,7 +33,7 @@ public function columns() 'name', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 100, 'notNull' => true, ] ), @@ -41,7 +41,7 @@ public function columns() 'display_name', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 255, 'notNull' => true, ] ), diff --git a/system/Base/Installer/Packages/Setup/Schema/Modules/Middlewares.php b/system/Base/Installer/Packages/Setup/Schema/Modules/Middlewares.php index 435c3be6f..833869eb0 100644 --- a/system/Base/Installer/Packages/Setup/Schema/Modules/Middlewares.php +++ b/system/Base/Installer/Packages/Setup/Schema/Modules/Middlewares.php @@ -25,7 +25,7 @@ public function columns() 'name', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 100, 'notNull' => true, ] ), @@ -33,7 +33,7 @@ public function columns() 'display_name', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 255, 'notNull' => true, ] ), @@ -65,7 +65,7 @@ public function columns() 'category', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 100, 'notNull' => true, ] ), diff --git a/system/Base/Installer/Packages/Setup/Schema/Modules/Packages.php b/system/Base/Installer/Packages/Setup/Schema/Modules/Packages.php index 66b7a9d8d..066c4fb98 100644 --- a/system/Base/Installer/Packages/Setup/Schema/Modules/Packages.php +++ b/system/Base/Installer/Packages/Setup/Schema/Modules/Packages.php @@ -25,7 +25,7 @@ public function columns() 'name', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 100, 'notNull' => true, ] ), @@ -33,7 +33,7 @@ public function columns() 'display_name', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 255, 'notNull' => true, ] ), @@ -65,7 +65,7 @@ public function columns() 'category', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 100, 'notNull' => true, ] ), diff --git a/system/Base/Installer/Packages/Setup/Schema/Modules/Views.php b/system/Base/Installer/Packages/Setup/Schema/Modules/Views.php index 80334687c..a76d70189 100644 --- a/system/Base/Installer/Packages/Setup/Schema/Modules/Views.php +++ b/system/Base/Installer/Packages/Setup/Schema/Modules/Views.php @@ -25,7 +25,7 @@ public function columns() 'name', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 100, 'notNull' => true, ] ), @@ -33,7 +33,7 @@ public function columns() 'display_name', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 255, 'notNull' => true, ] ), @@ -65,7 +65,7 @@ public function columns() 'category', [ 'type' => Column::TYPE_VARCHAR, - 'size' => 50, + 'size' => 100, 'notNull' => true, ] ), From b7765fd030318fa68041d28ed1ffb99b91487f0f Mon Sep 17 00:00:00 2001 From: "Gurdeep Singh (Guru)" Date: Tue, 25 Feb 2025 06:41:56 +1100 Subject: [PATCH 07/11] Initial commit #637 --- .../Devtools/Modules/ModulesComponent.php | 28 + .../Devtools/Test/Install/component.json | 99 ++-- .../Devtools/Modules/DevtoolsModules.php | 255 ++++++++- .../Modules/Install/Schema/FilesHash.php | 62 +++ .../Devtools/Modules/Install/package.json | 10 +- .../Model/AppsCoreDevtoolsFilesHash.php | 16 + .../html/devtools/modules/changes.html | 18 + .../html/devtools/modules/changes/view.html | 201 ++++++++ .../html/devtools/modules/repo/view.html | 11 +- .../Default/html/devtools/modules/select.html | 25 +- system/Base/Installer/Packages/Setup.php | 2 +- .../Base/Installer/Packages/Setup/Schema.php | 488 +++++++++--------- .../CoreServiceProvider/Install/Install.php | 2 +- .../DatabaseServiceProvider/Ff/Store.php | 4 + 14 files changed, 922 insertions(+), 299 deletions(-) create mode 100644 apps/Core/Packages/Devtools/Modules/Install/Schema/FilesHash.php create mode 100644 apps/Core/Packages/Devtools/Modules/Model/AppsCoreDevtoolsFilesHash.php create mode 100644 apps/Core/Views/Default/html/devtools/modules/changes.html create mode 100644 apps/Core/Views/Default/html/devtools/modules/changes/view.html diff --git a/apps/Core/Components/Devtools/Modules/ModulesComponent.php b/apps/Core/Components/Devtools/Modules/ModulesComponent.php index bf28933ee..15a1228bf 100644 --- a/apps/Core/Components/Devtools/Modules/ModulesComponent.php +++ b/apps/Core/Components/Devtools/Modules/ModulesComponent.php @@ -3,6 +3,10 @@ namespace Apps\Core\Components\Devtools\Modules; use Apps\Core\Packages\Devtools\Modules\DevtoolsModules; +use League\Flysystem\FilesystemException; +use League\Flysystem\UnableToCheckExistence; +use League\Flysystem\UnableToListContents; +use League\Flysystem\UnableToRetrieveMetadata; use System\Base\BaseComponent; use z4kn4fein\SemVer\Version; @@ -176,6 +180,30 @@ public function viewAction() unset($modules['views']); } } + } else if (isset($this->getData()['changes']) && + $this->getData()['changes'] == true + ) { + unset($modules['core']); + unset($modules['apptypes']); + unset($modules['bundles']); + + foreach ($modules as $moduleType => &$modulesTypeArr) { + if (isset($modulesTypeArr['childs']) && count($modulesTypeArr['childs']) > 0) { + foreach ($modulesTypeArr['childs'] as $childKey => &$child) { + $child = $this->modulesPackage->validateFilesHash($child); + + if ($child['repoExists'] && !$child['isModified']) { + unset($modules[$moduleType]['childs'][$childKey]); + } + } + } + } + + $this->view->modules = $modules; + + $this->view->pick('modules/changes'); + + return; } $this->view->modules = $modules; diff --git a/apps/Core/Components/Devtools/Test/Install/component.json b/apps/Core/Components/Devtools/Test/Install/component.json index f05292a1b..7e60a9450 100644 --- a/apps/Core/Components/Devtools/Test/Install/component.json +++ b/apps/Core/Components/Devtools/Test/Install/component.json @@ -1,45 +1,58 @@ { - "route" : "devtools/test", - "name" : "Test", - "description" : "Test anything in this component.", - "module_type" : "components", - "app_type" : "core", - "category" : "devtools", - "version" : "0.0.0", - "repo" : "https://.../dash-component-system-devtools-test", - "class" : "Apps\\Core\\Components\\Devtools\\Test\\TestComponent", - "dependencies" : { - "core" : { - "name" : "Core", - "version" : "0.0.0", - "repo" : "https://.../" - }, - "components" : [], - "packages" : - [ - { - "name" : "Test", - "version" : "0.0.0", - "repo" : "https://.../dash-package-system-devtools-test" - } - ], - "middlewares" : [], - "views" : [], - "external" : [] - }, - "menu" : { - "seq" : 99, - "devtools" : { - "title" : "devtools", - "icon" : "code", - "childs" : { - "test" : { - "title" : "test", - "link" : "devtools/test" - } - } - } - }, - "settings" : { - } + "name" : "Test", + "route" : "devtools/test", + "description" : "Test anything in this component.", + "module_type" : "components", + "app_type" : "core", + "category" : "devtools", + "version" : "0.0.0", + "repo" : "https://.../dash-component-system-devtools-test", + "class" : "Apps\\Core\\Components\\Devtools\\Test\\TestComponent", + "dependencies" : + { + "core" : + { + "name" : "Core", + "version" : "0.0.0", + "repo" : "https://.../" + }, + "apptype" : [], + "packages" : [], + "middlewares" : [], + "views" : [], + "externals" : + { + "composer" : + { + "require" : [] + }, + "config" : + { + "allow-plugins" : [] + }, + "extra" : + { + "patches" : [] + } + } + }, + "menu" : + { + "seq" : 99, + "devtools" : + { + "title" : "devtools", + "icon" : "code", + "childs" : + { + "test" : + { + "title" : "test", + "link" : "devtools/test" + } + } + } + }, + "settings" : [], + "widgets" : [] } \ No newline at end of file diff --git a/apps/Core/Packages/Devtools/Modules/DevtoolsModules.php b/apps/Core/Packages/Devtools/Modules/DevtoolsModules.php index 767af93b7..024acaa05 100644 --- a/apps/Core/Packages/Devtools/Modules/DevtoolsModules.php +++ b/apps/Core/Packages/Devtools/Modules/DevtoolsModules.php @@ -2,6 +2,7 @@ namespace Apps\Core\Packages\Devtools\Modules; +use Apps\Core\Packages\Devtools\Modules\Model\AppsCoreDevtoolsFilesHash; use Apps\Core\Packages\Devtools\Modules\Settings; use League\Flysystem\FilesystemException; use League\Flysystem\UnableToCheckExistence; @@ -20,6 +21,8 @@ class DevtoolsModules extends BasePackage { + protected $modelToUse = AppsCoreDevtoolsFilesHash::class; + protected $apiClient; protected $apiClientConfig; @@ -146,6 +149,10 @@ public function addModule($data) $this->updateModuleJson($data, false, $viewPublic) && $this->modules->{$data['module_type']}->add($data) ) { + if (strtolower($data['app_type']) !== 'core') { + $this->reCalculateFilesHash($this->modules->{$data['module_type']}->packagesData->last); + } + if ($data['createrepo'] == true) { if ($data['module_type'] === 'views' && $data['is_subview'] == false) {//Create public repository as well if (!$this->checkRepo($data)) { @@ -292,6 +299,10 @@ public function updateModule($data) if ($this->updateModuleJson($data, false, $viewPublic) && $this->modules->{$data['module_type']}->update($module) ) { + if (strtolower($data['app_type']) !== 'core') { + $this->reCalculateFilesHash($this->modules->{$data['module_type']}->packagesData->last); + } + if ($data['module_type'] === 'components') { $this->addUpdateComponentMenu($data); } @@ -448,6 +459,10 @@ public function removeModule($data) $this->modules->{$module['module_type']}->remove($module['id']); } + if ($data['module_type'] !== 'bundles') { + $this->reCalculateFilesHash($module, true); + } + $this->addResponse('Removed module from DB & files from the system...'); return true; @@ -458,6 +473,235 @@ public function removeModule($data) } } + protected function getFilesHash($module) + { + if ($this->config->databasetype === 'db') { + $conditions = + [ + 'conditions' => 'module_type = :module_type: AND module_id = :module_id:', + 'bind' => + [ + 'module_type' => $module['module_type'], + 'module_id' => $module['id'] + ] + ]; + } else { + $conditions = + [ + 'conditions' => [ + ['module_type', '=', $module['module_type']], + ['module_id', '=', $module['id']] + ] + ]; + } + + $filesHash = $this->getByParams($conditions); + if ($filesHash && isset($filesHash[0])) { + $filesHash = $filesHash[0]; + } + + return $filesHash; + } + + public function reCalculateFilesHash($module, $remove = false, $viaGenerateRelease = false, $viaValidation = false) + { + if (!$viaValidation) { + $filesHash = $this->getFilesHash($module); + + if ($filesHash && $remove) { + $this->remove($filesHash['id']); + + return true; + } + } else { + $filesHash = false; + } + + if (!$filesHash || $viaGenerateRelease) {//We only generate hash when there is no entry or when we generate a new release + if (!$viaGenerateRelease) { + $filesHash = []; + $filesHash['module_type'] = $module['module_type']; + $filesHash['module_id'] = $module['id']; + } + + $moduleLocation = $this->getModuleFilesLocation($module); + + $moduleLocationFiles = $this->basepackages->utils->scanDir($moduleLocation, true, ['.git/', '.fileHashes']); + + //In case you have .gitignore file, it can get complicated to sort which files to hash. + //So, the files you want to hash, just add them in .fileHashes in the same folder as .gitignore + //The file should have a list of all files to hash (1 filename per line) + //Example: + //.gitignore + // README.md + // view.html + // view.json + $filesToHash = []; + if ($this->localContent->fileExists($moduleLocation . '.fileHashes')) { + $filesToHash = $this->localContent->read($moduleLocation . '.fileHashes'); + + if ($filesToHash && $filesToHash !== '') { + $filesToHash = str_replace("'", '', $filesToHash); + $filesToHash = explode(PHP_EOL, $filesToHash); + } + } + + if ($module['module_type'] === 'views' && $module['is_subview'] == false) { + $moduleLocation = $this->getModuleFilesLocation($module, true); + + $viewFiles = $this->basepackages->utils->scanDir($moduleLocation, true, ['.git/', '.fileHashes']); + + if ($viewFiles && count($viewFiles['files']) > 0) { + $moduleLocationFiles['files'] = array_merge($moduleLocationFiles['files'], $viewFiles['files']); + } + + if ($this->localContent->fileExists($moduleLocation . '.fileHashes')) { + $viewFilesToHash = $this->localContent->read($moduleLocation . '.fileHashes'); + + if ($viewFilesToHash && $viewFilesToHash !== '') { + $viewFilesToHash = str_replace("'", '', $viewFilesToHash); + $viewFilesToHash = explode(PHP_EOL, $viewFilesToHash); + + if (count($viewFilesToHash) > 0) { + $filesToHash = array_merge($filesToHash, $viewFilesToHash); + } + } + } + } + + $filesHash['files_hash'] = []; + + if ($moduleLocationFiles && count($moduleLocationFiles['files']) > 0) { + foreach ($moduleLocationFiles['files'] as $file) { + $filePath = $file; + + $file = str_replace($moduleLocation, '', $file); + + if (count($filesToHash) > 0) { + if (!in_array($file, $filesToHash)) { + continue; + } + } + + $hash = hash_file('md5', base_path($filePath)); + + $filesHash['files_hash'][$file] = $hash; + } + } + + if (isset($filesHash['id'])) { + $this->update($filesHash); + } else { + $this->add($filesHash); + } + } + + return $filesHash; + } + + public function validateFilesHash($module) + { + try { + $moduleLocation = $this->getNewFilesLocation($module); + + $module['repoExists'] = false; + if ($this->localContent->directoryExists($moduleLocation . '.git')) { + $module['repoExists'] = true; + + if (!isset($module['repo_details'])) { + $this->modules->manager->getModuleInfo( + [ + 'module_type' => $module['module_type'], + 'module_id' => $module['id'], + 'sync' => true + ] + ); + } + } + + $filesHash = $this->getFilesHash($module); + + $module['isModified'] = false; + if (!$filesHash) { + $this->reCalculateFilesHash($module, false, false, true); + + return $module; + } + + $moduleFiles = $this->basepackages->utils->scanDir($moduleLocation, true, ['.git/', '.fileHashes']); + + //In case you have .gitignore file, it can get complicated to sort which files to hash. + //So, the files you want to hash, just add them in .fileHashes in the same folder as .gitignore + //The file should have a list of all files to hash (1 filename per line) + //Example: + //.gitignore + // README.md + // view.html + // view.json + $filesToHash = []; + if ($this->localContent->fileExists($moduleLocation . '.fileHashes')) { + $filesToHash = $this->localContent->read($moduleLocation . '.fileHashes'); + + if ($filesToHash && $filesToHash !== '') { + $filesToHash = str_replace("'", '', $filesToHash); + $filesToHash = explode(PHP_EOL, $filesToHash); + } + } + + if ($module['module_type'] === 'views' && $module['is_subview'] == false) { + $moduleLocation = $this->getModuleFilesLocation($module, true); + + $viewFiles = $this->basepackages->utils->scanDir($moduleLocation, true, ['.git/', '.fileHashes']); + + if ($viewFiles && count($viewFiles['files']) > 0) { + $moduleFiles['files'] = array_merge($moduleFiles['files'], $viewFiles['files']); + } + + if ($this->localContent->fileExists($moduleLocation . '.fileHashes')) { + $viewFilesToHash = $this->localContent->read($moduleLocation . '.fileHashes'); + + if ($viewFilesToHash && $viewFilesToHash !== '') { + $viewFilesToHash = str_replace("'", '', $viewFilesToHash); + $viewFilesToHash = explode(PHP_EOL, $viewFilesToHash); + + if (count($viewFilesToHash) > 0) { + $filesToHash = array_merge($filesToHash, $viewFilesToHash); + } + } + } + } + + if ($moduleFiles && count($moduleFiles['files']) > 0) { + foreach ($moduleFiles['files'] as $file) { + $filePath = $file; + + $file = str_replace($moduleLocation, '', $file); + + if (count($filesToHash) > 0) { + if (!in_array($file, $filesToHash)) { + continue; + } + } + + $hash = hash_file('md5', base_path($filePath)); + + if (!isset($filesHash['files_hash'][$file]) || + (isset($filesHash['files_hash'][$file]) && + $filesHash['files_hash'][$file] !== $hash) + ) { + $module['isModified'] = true; + + return $module; + } + } + } + } catch (\throwable | FilesystemException | UnableToCheckExistence $e) { + throw $e; + } + + return $module; + } + public function cleanup($modulePath) { $files = $this->basepackages->utils->scanDir($modulePath); @@ -747,6 +991,8 @@ protected function checkAppType($data) $this->apps->types->add($data); + $this->apps->types->packagesData->last; + $this->addUpdateAppTypeFiles($data); } @@ -1231,7 +1477,7 @@ protected function generateNewFiles(&$data) return true; } - protected function getNewFilesLocation($data, $viewPublic = false) + public function getNewFilesLocation($data, $viewPublic = false) { if ($data['module_type'] === 'components') { $moduleLocation = 'apps/' . ucfirst($data['app_type']) . '/Components/'; @@ -2694,6 +2940,10 @@ public function generateRelease($data) } if (count($releaseData) > 0 && isset($releaseData['newRelease'])) { + if ($module['app_type'] !== 'core' && $module['module_type'] !== 'bundles' && $module['module_type'] !== 'apptypes') { + $this->reCalculateFilesHash($module, false, true); + } + $this->addResponse('Generated New Release!', 0, $releaseData); return true; @@ -3585,7 +3835,8 @@ public function getRemoteModules($id) if (in_array(strtolower($names[1]), $modulesTypeArr)) { $module['module_type'] = $names[1]; } else { - $module['module_type'] = '-'; + unset($modulesArr[$key]); + continue; } } diff --git a/apps/Core/Packages/Devtools/Modules/Install/Schema/FilesHash.php b/apps/Core/Packages/Devtools/Modules/Install/Schema/FilesHash.php new file mode 100644 index 000000000..3703f01fa --- /dev/null +++ b/apps/Core/Packages/Devtools/Modules/Install/Schema/FilesHash.php @@ -0,0 +1,62 @@ + [ + new Column( + 'id', + [ + 'type' => Column::TYPE_INTEGER, + 'notNull' => true, + 'autoIncrement' => true, + 'primary' => true, + ] + ), + new Column( + 'module_type', + [ + 'type' => Column::TYPE_VARCHAR, + 'size' => 50, + 'notNull' => true, + ] + ), + new Column( + 'module_id', + [ + 'type' => Column::TYPE_INTEGER, + 'notNull' => true, + ] + ), + new Column( + 'files_hash', + [ + 'type' => Column::TYPE_JSON, + 'notNull' => true, + ] + ) + ], + 'indexes' => [ + new Index( + 'column_UNIQUE', + [ + 'module_type', + 'module_id' + ], + 'UNIQUE' + ) + ], + 'options' => [ + 'TABLE_COLLATION' => 'utf8mb4_general_ci' + ] + ]; + } +} diff --git a/apps/Core/Packages/Devtools/Modules/Install/package.json b/apps/Core/Packages/Devtools/Modules/Install/package.json index 9786533c1..a27ce65f3 100644 --- a/apps/Core/Packages/Devtools/Modules/Install/package.json +++ b/apps/Core/Packages/Devtools/Modules/Install/package.json @@ -20,9 +20,13 @@ "packages" : [], "middlewares" : [], "views" : [], - "external" : [] + "external" : [], + "externals" : [] }, - "settings" : { - "api_clients" : [1,2] + "settings" : + { + "api_clients" : [1, + 2 + ] } } \ No newline at end of file diff --git a/apps/Core/Packages/Devtools/Modules/Model/AppsCoreDevtoolsFilesHash.php b/apps/Core/Packages/Devtools/Modules/Model/AppsCoreDevtoolsFilesHash.php new file mode 100644 index 000000000..6cb53e74a --- /dev/null +++ b/apps/Core/Packages/Devtools/Modules/Model/AppsCoreDevtoolsFilesHash.php @@ -0,0 +1,16 @@ + +{% set changesHtml = '' %} +{% if modules|length > 0 %} + {% for moduleType, modulesArr in modules %} + {% if modulesArr['childs'] is defined and modulesArr['childs']|length > 0 %} + {% for moduleArr in modulesArr['childs'] %} + {% set url = links.url('devtools/modules/q/module/true/type/' ~ moduleArr['module_type'] ~ '/id/' ~ moduleArr['id']) %} + {% set moduleEdit = + '' + %} + {% set moduleUrl = '-' %} + {% if moduleArr['repoExists'] != false and moduleArr['repo_details']['details']['html_url'] is defined %} + {% set moduleUrl = '' ~ moduleArr['repo_details']['details']['html_url'] %} + {% endif %} + {% set moduleModified = '-' %} + {% if moduleArr['isModified'] != false %} + {% set moduleModified = 'Yes' %} + {% endif %} + {% set changesHtml = changesHtml ~ '' ~ + ' ' ~ moduleArr['name'] ~ '' ~ + ' ' ~ moduleArr['module_type'] ~ '' ~ + ' ' ~ moduleUrl ~ '' ~ + ' ' ~ moduleModified ~ '' ~ + ' ' ~ moduleEdit ~ '' ~ + ' ' %} + {% endfor %} + {% endif %} + {% endfor %} +{% endif %} +{% if changesHtml != '' %} +
+
+ {{adminltetags.useTag('buttons', + [ + 'component' : component, + 'componentName' : componentName, + 'componentId' : componentId, + 'sectionId' : sectionId, + 'buttonType' : 'button', + 'buttons' : + [ + 'modules-link' : [ + 'title' : 'link', + 'size' : 'xs', + 'disabled' : true, + 'hidden' : true, + 'url' : '#' + ] + ] + ] + )}} +
+
+
+
+ + + + + + + + + + + + {{changesHtml}} + +
NameModule TypeRepositoryModifiedEdit
+
+
+{% endif %} +{% set modules = json_encode(modules, 16) %} + diff --git a/apps/Core/Views/Default/html/devtools/modules/repo/view.html b/apps/Core/Views/Default/html/devtools/modules/repo/view.html index f53050b51..8062ea5e1 100644 --- a/apps/Core/Views/Default/html/devtools/modules/repo/view.html +++ b/apps/Core/Views/Default/html/devtools/modules/repo/view.html @@ -1,4 +1,4 @@ - + {% set repoHtml = '' %} {% if remoteModules|length > 0 %} {% for remoteModule in remoteModules %} @@ -53,7 +53,7 @@
-
+
@@ -70,12 +70,6 @@
-
-
-
- -
-
{% endif %} {% set remoteModules = json_encode(remoteModules, 16) %} - \ No newline at end of file diff --git a/apps/Core/Views/Default/html/devtools/modules/select.html b/apps/Core/Views/Default/html/devtools/modules/select.html index 85b8eb608..88672f422 100644 --- a/apps/Core/Views/Default/html/devtools/modules/select.html +++ b/apps/Core/Views/Default/html/devtools/modules/select.html @@ -63,6 +63,21 @@ )}}
+ {{adminltetags.useTag('buttons', + [ + 'componentId' : componentId, + 'sectionId' : sectionId, + 'buttonType' : 'button', + 'buttons' : + [ + 'check-for-changes' : [ + 'title' : 'Check for module changes', + 'type' : 'primary', + 'icon' : 'code' + ] + ] + ] + )}} {{adminltetags.useTag('buttons', [ 'componentId' : componentId, @@ -475,6 +490,11 @@ $('#{{componentId}}-{{sectionId}}-commit-bundle-json').click(function() { dataCollectionSectionForm['funcs']['loadAjax'](id, label, false, false, false, true); }); + + $('#{{componentId}}-{{sectionId}}-check-for-changes').off(); + $('#{{componentId}}-{{sectionId}}-check-for-changes').click(function() { + dataCollectionSectionForm['funcs']['loadAjax'](null, null, false, false, false, false, false, false, true); + }); } }, '{{componentId}}-{{sectionId}}-form' : { @@ -486,7 +506,7 @@ }); dataCollectionSectionForm['funcs']['loadAjax'] = - function(id = null, label = null, includecoremodules = false, newrelease = false, clone = false, bundlesjson = false, isSubView = false, isRemote = false) { + function(id = null, label = null, includecoremodules = false, newrelease = false, clone = false, bundlesjson = false, isSubView = false, isRemote = false, isChanges = false) { if (id && label) { if (label === 'bundles' && !newrelease) { dataCollectionSectionForm['vars']['url'] = '{{links.url("devtools/modules/q/bundles/true/id/' + id + '")}}'; @@ -525,6 +545,9 @@ if (id && isRemote) { dataCollectionSectionForm['vars']['url'] = '{{links.url("devtools/modules/q/repo/true/id/' + id + '")}}'; } + if (isChanges) { + dataCollectionSectionForm['vars']['url'] = '{{links.url("devtools/modules/q/changes/true")}}'; + } if (dataCollectionSectionForm['vars']['url'].charAt(0) !== 'h' && dataCollectionSectionForm['vars']['url'].charAt(0) !== '/' diff --git a/system/Base/Installer/Packages/Setup.php b/system/Base/Installer/Packages/Setup.php index 49d2799fd..b91cf62c4 100644 --- a/system/Base/Installer/Packages/Setup.php +++ b/system/Base/Installer/Packages/Setup.php @@ -359,7 +359,7 @@ protected function checkDbEmpty() protected function buildSchema() { - $databases = (new Schema)->getSchema(); + $databases = (new Schema)->getSchema($this->postData['dev']); if (isset($this->postData['databasetype']) && $this->postData['databasetype'] !== 'ff') { foreach ($databases as $tableName => $tableClass) { diff --git a/system/Base/Installer/Packages/Setup/Schema.php b/system/Base/Installer/Packages/Setup/Schema.php index 861281279..e093d9fa0 100644 --- a/system/Base/Installer/Packages/Setup/Schema.php +++ b/system/Base/Installer/Packages/Setup/Schema.php @@ -122,245 +122,255 @@ class Schema { - public function getSchema() + public function getSchema($dev) { - return [ - 'service_provider_core' => [ - 'schema' => new Core, - 'model' => new ServiceProviderCore, - ], - 'service_provider_apps' => [ - 'schema' => new Apps, - 'model' => new ServiceProviderApps, - ], - 'service_provider_apps_types' => [ - 'schema' => new Types, - 'model' => new ServiceProviderAppsTypes, - ], - 'service_provider_access_ip_filter' => [ - 'schema' => new IpFilter, - 'model' => new ServiceProviderAccessIpFilter, - ], - 'service_provider_domains' => [ - 'schema' => new Domains, - 'model' => new ServiceProviderDomains, - ], - 'service_provider_modules_queues' => [ - 'schema' => new Queues, - 'model' => new ServiceProviderModulesQueues, - ], - 'modules_bundles' => [ - 'schema' => new Bundles, - 'model' => new ModulesBundles, - ], - 'modules_components' => [ - 'schema' => new Components, - 'model' => new ModulesComponents, - ], - 'modules_packages' => [ - 'schema' => new Packages, - 'model' => new ModulesPackages, - ], - 'modules_middlewares' => [ - 'schema' => new Middlewares, - 'model' => new ModulesMiddlewares, - ], - 'modules_views' => [ - 'schema' => new Views, - 'model' => new ModulesViews, - ], - 'modules_views_settings' => [ - 'schema' => new Settings, - 'model' => new ModulesViewsSettings, - ], - 'modules_externals' => [ - 'schema' => new Externals, - 'model' => new ModulesExternals, - ], - 'basepackages_email_services' => [ - 'schema' => new EmailServices, - 'model' => new BasepackagesEmailServices, - ], - 'basepackages_email_queue' => [ - 'schema' => new EmailQueue, - 'model' => new BasepackagesEmailQueue, - ], - 'basepackages_users_accounts' => [ - 'schema' => new Accounts, - 'model' => new BasepackagesUsersAccounts, - ], - 'basepackages_users_accounts_security' => [ - 'schema' => new Security, - 'model' => new BasepackagesUsersAccountsSecurity, - ], - 'basepackages_users_accounts_canlogin' => [ - 'schema' => new CanLogin, - 'model' => new BasepackagesUsersAccountsCanlogin, - ], - 'basepackages_users_accounts_sessions' => [ - 'schema' => new Sessions, - 'model' => new BasepackagesUsersAccountsSessions, - ], - 'basepackages_users_accounts_identifiers' => [ - 'schema' => new Identifiers, - 'model' => new BasepackagesUsersAccountsIdentifiers, - ], - 'basepackages_users_accounts_agents' => [ - 'schema' => new Agents, - 'model' => new BasepackagesUsersAccountsAgents, - ], - 'basepackages_users_accounts_tunnels' => [ - 'schema' => new Tunnels, - 'model' => new BasepackagesUsersAccountsTunnels, - ], - 'basepackages_users_profiles' => [ - 'schema' => new Profiles, - 'model' => new BasepackagesUsersProfiles, - ], - 'basepackages_users_roles' => [ - 'schema' => new Roles, - 'model' => new BasepackagesUsersRoles, - ], - 'basepackages_menus' => [ - 'schema' => new Menus, - 'model' => new BasepackagesMenus, - ], - 'basepackages_murls' => [ - 'schema' => new Murls, - 'model' => new BasepackagesMurls, - ], - 'basepackages_filters' => [ - 'schema' => new Filters, - 'model' => new BasepackagesFilters, - ], - 'basepackages_geo_countries' => [ - 'schema' => new Countries, - 'model' => new BasepackagesGeoCountries, - ], - 'basepackages_geo_states' => [ - 'schema' => new States, - 'model' => new BasepackagesGeoStates, - ], - 'basepackages_geo_cities' => [ - 'schema' => new Cities, - 'model' => new BasepackagesGeoCities, - ], - 'basepackages_geo_cities_ip2locationv4' => [ - 'schema' => new CitiesIp2LocationV4, - 'model' => new BasepackagesGeoCitiesIp2locationv4, - ], - 'basepackages_geo_cities_ip2locationv6' => [ - 'schema' => new CitiesIp2LocationV6, - 'model' => new BasepackagesGeoCitiesIp2locationv6, - ], - 'basepackages_geo_timezones' => [ - 'schema' => new Timezones, - 'model' => new BasepackagesGeoTimezones, - ], - 'basepackages_address_book' => [ - 'schema' => new AddressBook, - 'model' => new BasepackagesAddressBook, - ], - 'basepackages_storages' => [ - 'schema' => new Storages, - 'model' => new BasepackagesStorages, - ], - 'basepackages_storages_local' => [ - 'schema' => new StoragesLocal, - 'model' => new BasepackagesStoragesLocal, - ], - 'basepackages_activity_logs' => [ - 'schema' => new ActivityLogs, - 'model' => new BasepackagesActivityLogs, - ], - 'basepackages_notes' => [ - 'schema' => new Notes, - 'model' => new BasepackagesNotes, - ], - 'basepackages_notifications' => [ - 'schema' => new Notifications, - 'model' => new BasepackagesNotifications, - ], - 'basepackages_workers_workers' => [ - 'schema' => new Workers, - 'model' => new BasepackagesWorkersWorkers, - ], - 'basepackages_workers_schedules' => [ - 'schema' => new Schedules, - 'model' => new BasepackagesWorkersSchedules, - ], - 'basepackages_workers_tasks' => [ - 'schema' => new Tasks, - 'model' => new BasepackagesWorkersTasks, - ], - 'basepackages_workers_jobs' => [ - 'schema' => new Jobs, - 'model' => new BasepackagesWorkersJobs, - ], - 'basepackages_import_export' => [ - 'schema' => new ImportExport, - 'model' => new BasepackagesImportExport, - ], - 'basepackages_templates' => [ - 'schema' => new Templates, - 'model' => new BasepackagesTemplates, - ], - 'basepackages_dashboards' => [ - 'schema' => new Dashboards, - 'model' => new BasepackagesDashboards, - ], - 'basepackages_dashboards_widgets' => [ - 'schema' => new DashboardsWidgets, - 'model' => new BasepackagesDashboardsWidgets, - ], - 'basepackages_widgets' => [ - 'schema' => new Widgets, - 'model' => new BasepackagesWidgets, - ], - 'basepackages_messenger' => [ - 'schema' => new Messenger, - 'model' => new BasepackagesMessenger, - ], - 'basepackages_api_client_services' => [ - 'schema' => new ApiClientServices, - 'model' => new BasepackagesApiClientServices, - ], - 'basepackages_api_client_services_calls' => [ - 'schema' => new ApiClientServicesCalls, - 'model' => new BasepackagesApiClientServicesCalls, - ], - 'basepackages_api_client_services_apis_repos'=> [ - 'schema' => new Repos, - 'model' => null - ], - 'service_provider_api' => [ - 'schema' => new SPApi, - 'model' => new ServiceProviderApi, - ], - 'service_provider_api_access_tokens' => [ - 'schema' => new AccessTokens, - 'model' => new ServiceProviderApiAccessTokens, - ], - 'service_provider_api_authorization_codes' => [ - 'schema' => new AuthorizationCodes, - 'model' => new ServiceProviderApiAuthorizationCodes, - ], - 'service_provider_api_clients' => [ - 'schema' => new Clients, - 'model' => new ServiceProviderApiClients, - ], - 'service_provider_api_refresh_tokens' => [ - 'schema' => new RefreshTokens, - 'model' => new ServiceProviderApiRefreshTokens, - ], - 'service_provider_api_scopes' => [ - 'schema' => new Scopes, - 'model' => new ServiceProviderApiScopes, - ], - 'service_provider_api_scopes' => [ - 'schema' => new Scopes, - 'model' => new ServiceProviderApiScopes, - ] - ]; + $schema = + [ + 'service_provider_core' => [ + 'schema' => new Core, + 'model' => new ServiceProviderCore, + ], + 'service_provider_apps' => [ + 'schema' => new Apps, + 'model' => new ServiceProviderApps, + ], + 'service_provider_apps_types' => [ + 'schema' => new Types, + 'model' => new ServiceProviderAppsTypes, + ], + 'service_provider_access_ip_filter' => [ + 'schema' => new IpFilter, + 'model' => new ServiceProviderAccessIpFilter, + ], + 'service_provider_domains' => [ + 'schema' => new Domains, + 'model' => new ServiceProviderDomains, + ], + 'service_provider_modules_queues' => [ + 'schema' => new Queues, + 'model' => new ServiceProviderModulesQueues, + ], + 'modules_bundles' => [ + 'schema' => new Bundles, + 'model' => new ModulesBundles, + ], + 'modules_components' => [ + 'schema' => new Components, + 'model' => new ModulesComponents, + ], + 'modules_packages' => [ + 'schema' => new Packages, + 'model' => new ModulesPackages, + ], + 'modules_middlewares' => [ + 'schema' => new Middlewares, + 'model' => new ModulesMiddlewares, + ], + 'modules_views' => [ + 'schema' => new Views, + 'model' => new ModulesViews, + ], + 'modules_views_settings' => [ + 'schema' => new Settings, + 'model' => new ModulesViewsSettings, + ], + 'modules_externals' => [ + 'schema' => new Externals, + 'model' => new ModulesExternals, + ], + 'basepackages_email_services' => [ + 'schema' => new EmailServices, + 'model' => new BasepackagesEmailServices, + ], + 'basepackages_email_queue' => [ + 'schema' => new EmailQueue, + 'model' => new BasepackagesEmailQueue, + ], + 'basepackages_users_accounts' => [ + 'schema' => new Accounts, + 'model' => new BasepackagesUsersAccounts, + ], + 'basepackages_users_accounts_security' => [ + 'schema' => new Security, + 'model' => new BasepackagesUsersAccountsSecurity, + ], + 'basepackages_users_accounts_canlogin' => [ + 'schema' => new CanLogin, + 'model' => new BasepackagesUsersAccountsCanlogin, + ], + 'basepackages_users_accounts_sessions' => [ + 'schema' => new Sessions, + 'model' => new BasepackagesUsersAccountsSessions, + ], + 'basepackages_users_accounts_identifiers' => [ + 'schema' => new Identifiers, + 'model' => new BasepackagesUsersAccountsIdentifiers, + ], + 'basepackages_users_accounts_agents' => [ + 'schema' => new Agents, + 'model' => new BasepackagesUsersAccountsAgents, + ], + 'basepackages_users_accounts_tunnels' => [ + 'schema' => new Tunnels, + 'model' => new BasepackagesUsersAccountsTunnels, + ], + 'basepackages_users_profiles' => [ + 'schema' => new Profiles, + 'model' => new BasepackagesUsersProfiles, + ], + 'basepackages_users_roles' => [ + 'schema' => new Roles, + 'model' => new BasepackagesUsersRoles, + ], + 'basepackages_menus' => [ + 'schema' => new Menus, + 'model' => new BasepackagesMenus, + ], + 'basepackages_murls' => [ + 'schema' => new Murls, + 'model' => new BasepackagesMurls, + ], + 'basepackages_filters' => [ + 'schema' => new Filters, + 'model' => new BasepackagesFilters, + ], + 'basepackages_geo_countries' => [ + 'schema' => new Countries, + 'model' => new BasepackagesGeoCountries, + ], + 'basepackages_geo_states' => [ + 'schema' => new States, + 'model' => new BasepackagesGeoStates, + ], + 'basepackages_geo_cities' => [ + 'schema' => new Cities, + 'model' => new BasepackagesGeoCities, + ], + 'basepackages_geo_cities_ip2locationv4' => [ + 'schema' => new CitiesIp2LocationV4, + 'model' => new BasepackagesGeoCitiesIp2locationv4, + ], + 'basepackages_geo_cities_ip2locationv6' => [ + 'schema' => new CitiesIp2LocationV6, + 'model' => new BasepackagesGeoCitiesIp2locationv6, + ], + 'basepackages_geo_timezones' => [ + 'schema' => new Timezones, + 'model' => new BasepackagesGeoTimezones, + ], + 'basepackages_address_book' => [ + 'schema' => new AddressBook, + 'model' => new BasepackagesAddressBook, + ], + 'basepackages_storages' => [ + 'schema' => new Storages, + 'model' => new BasepackagesStorages, + ], + 'basepackages_storages_local' => [ + 'schema' => new StoragesLocal, + 'model' => new BasepackagesStoragesLocal, + ], + 'basepackages_activity_logs' => [ + 'schema' => new ActivityLogs, + 'model' => new BasepackagesActivityLogs, + ], + 'basepackages_notes' => [ + 'schema' => new Notes, + 'model' => new BasepackagesNotes, + ], + 'basepackages_notifications' => [ + 'schema' => new Notifications, + 'model' => new BasepackagesNotifications, + ], + 'basepackages_workers_workers' => [ + 'schema' => new Workers, + 'model' => new BasepackagesWorkersWorkers, + ], + 'basepackages_workers_schedules' => [ + 'schema' => new Schedules, + 'model' => new BasepackagesWorkersSchedules, + ], + 'basepackages_workers_tasks' => [ + 'schema' => new Tasks, + 'model' => new BasepackagesWorkersTasks, + ], + 'basepackages_workers_jobs' => [ + 'schema' => new Jobs, + 'model' => new BasepackagesWorkersJobs, + ], + 'basepackages_import_export' => [ + 'schema' => new ImportExport, + 'model' => new BasepackagesImportExport, + ], + 'basepackages_templates' => [ + 'schema' => new Templates, + 'model' => new BasepackagesTemplates, + ], + 'basepackages_dashboards' => [ + 'schema' => new Dashboards, + 'model' => new BasepackagesDashboards, + ], + 'basepackages_dashboards_widgets' => [ + 'schema' => new DashboardsWidgets, + 'model' => new BasepackagesDashboardsWidgets, + ], + 'basepackages_widgets' => [ + 'schema' => new Widgets, + 'model' => new BasepackagesWidgets, + ], + 'basepackages_messenger' => [ + 'schema' => new Messenger, + 'model' => new BasepackagesMessenger, + ], + 'basepackages_api_client_services' => [ + 'schema' => new ApiClientServices, + 'model' => new BasepackagesApiClientServices, + ], + 'basepackages_api_client_services_calls' => [ + 'schema' => new ApiClientServicesCalls, + 'model' => new BasepackagesApiClientServicesCalls, + ], + 'basepackages_api_client_services_apis_repos'=> [ + 'schema' => new Repos, + 'model' => null + ], + 'service_provider_api' => [ + 'schema' => new SPApi, + 'model' => new ServiceProviderApi, + ], + 'service_provider_api_access_tokens' => [ + 'schema' => new AccessTokens, + 'model' => new ServiceProviderApiAccessTokens, + ], + 'service_provider_api_authorization_codes' => [ + 'schema' => new AuthorizationCodes, + 'model' => new ServiceProviderApiAuthorizationCodes, + ], + 'service_provider_api_clients' => [ + 'schema' => new Clients, + 'model' => new ServiceProviderApiClients, + ], + 'service_provider_api_refresh_tokens' => [ + 'schema' => new RefreshTokens, + 'model' => new ServiceProviderApiRefreshTokens, + ], + 'service_provider_api_scopes' => [ + 'schema' => new Scopes, + 'model' => new ServiceProviderApiScopes, + ], + 'service_provider_api_scopes' => [ + 'schema' => new Scopes, + 'model' => new ServiceProviderApiScopes, + ] + ]; + + if ($dev == true) { + $schema['apps_core_devtools_files_hash'] = [ + 'schema' => new \Apps\Core\Packages\Devtools\Modules\Install\Schema\FilesHash, + 'model' => new \Apps\Core\Packages\Devtools\Modules\Model\AppsCoreDevtoolsFilesHash, + ]; + } + + return $schema; } } \ No newline at end of file diff --git a/system/Base/Providers/CoreServiceProvider/Install/Install.php b/system/Base/Providers/CoreServiceProvider/Install/Install.php index 29a458661..4c9cf2c5e 100644 --- a/system/Base/Providers/CoreServiceProvider/Install/Install.php +++ b/system/Base/Providers/CoreServiceProvider/Install/Install.php @@ -14,7 +14,7 @@ class Install extends BasePackage public function init($schemaNames = []) { - $databases = (new Schema)->getSchema(); + $databases = (new Schema)->getSchema($this->core->core['settings']['dev']); $this->databases = $databases; diff --git a/system/Base/Providers/DatabaseServiceProvider/Ff/Store.php b/system/Base/Providers/DatabaseServiceProvider/Ff/Store.php index 53420595e..8a02c3425 100644 --- a/system/Base/Providers/DatabaseServiceProvider/Ff/Store.php +++ b/system/Base/Providers/DatabaseServiceProvider/Ff/Store.php @@ -1714,6 +1714,10 @@ protected function decreaseCounter() protected function updateCounters($counters) { + if ($counters['totalEntries'] === 0) { + $counters['lastId'] = 0; + } + $updateCounters = IoHelper::updateFileContent( $this->storePath . '_cnt.sdb', function () use ($counters) { From fe6556d3bd626dfe8faaa2a6d32b64f0999dc58c Mon Sep 17 00:00:00 2001 From: "Gurdeep Singh (Guru)" Date: Tue, 25 Feb 2025 13:46:41 +1100 Subject: [PATCH 08/11] fix issue #634 --- apps/Core/Views/Default/html/modules/modules.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/Core/Views/Default/html/modules/modules.html b/apps/Core/Views/Default/html/modules/modules.html index 938d14d34..9b12d48f8 100644 --- a/apps/Core/Views/Default/html/modules/modules.html +++ b/apps/Core/Views/Default/html/modules/modules.html @@ -7,7 +7,6 @@ 'tooltip' : 'Repo Add', 'icon' : 'circle-plus', 'iconColor' : 'success', - 'disabled' : true, 'url' : links.url('system/api/client/services') ], 'repo-update' : @@ -15,7 +14,7 @@ 'id' : 'repo-update', 'tooltip' : 'Repo Update', 'icon' : 'edit', - 'iconColor' : 'warning', + 'iconColor' : 'secondary', 'disabled' : true, 'url' : links.url('system/api/client/services/q/id/') ], From 634aaf532a7f6c32300659807a753b90c69ed28d Mon Sep 17 00:00:00 2001 From: "Gurdeep Singh (Guru)" Date: Tue, 25 Feb 2025 19:43:19 +1100 Subject: [PATCH 09/11] initial commit #628 --- .../Devtools/Test/TestComponent.php | 75 ++++++--- .../System/Progress/ProgressComponent.php | 28 ++++ .../Default/html/devtools/test/view.html | 53 +++++- .../default/js/footer/core/Baz/BazProgress.js | 122 ++++++++++++-- public/core/default/js/footer/jsFooterCore.js | 122 ++++++++++++-- .../Packages/Progress.php | 158 ++++++++++++++---- .../DispatcherServiceProvider/Dispatcher.php | 4 + .../Providers/ErrorServiceProvider/Error.php | 4 + .../ExceptionHandlers.php | 7 + 9 files changed, 494 insertions(+), 79 deletions(-) diff --git a/apps/Core/Components/Devtools/Test/TestComponent.php b/apps/Core/Components/Devtools/Test/TestComponent.php index 4a34c9334..83d76d486 100644 --- a/apps/Core/Components/Devtools/Test/TestComponent.php +++ b/apps/Core/Components/Devtools/Test/TestComponent.php @@ -3,37 +3,24 @@ namespace Apps\Core\Components\Devtools\Test; use Apps\Core\Packages\Devtools\DicExtractData\DevtoolsDicExtractData; +use Apps\Core\Packages\Devtools\Test\DevtoolsTest; use System\Base\BaseComponent; class TestComponent extends BaseComponent { + protected $testPackage; + + public function initialize() + { + $this->testPackage = new DevtoolsTest; + } + /** * @acl(name=view) */ public function viewAction() { - // $install = new \Apps\Fintech\Components\Dashboards\Install\Install; - - // $install->init()->install(); - // $adminComponents = $this->basepackages->utils->scanDir('apps/Core/Components/', true); - - // foreach ($adminComponents['files'] as $adminComponentKey => $adminComponent) { - // if (strpos($adminComponent, 'component.json')) { - // try { - // $jsonFile = - // $this->helper->decode( - // $this->localContent->read($adminComponent), - // true - // ); - // } catch (\throwable $e) { - // throw new \Exception($e->getMessage() . '. Problem reading component.json at location ' . $adminComponent); - // } - - // if ($jsonFile['menu'] && $jsonFile['menu'] !== 'false') { - // $this->basepackages->menus->addMenu($jsonFile); - // } - // } - // } + return; } /** @@ -82,4 +69,48 @@ public function apiRemoveAction() { $this->addResponse('Test', 0, ['remove' => true]); } + + public function testAction() + { + if ($this->basepackages->progress->checkProgressFile()) { + $this->basepackages->progress->deleteProgressFile(); + } + + $this->basepackages->progress->registerMethods( + [ + [ + 'method' => 'testTest', + 'text' => 'Test', + ], + [ + 'method' => 'testTest', + 'text' => 'Test', + ], + [ + 'method' => 'testTest', + 'text' => 'Test', + ], + [ + 'method' => 'testDownload', + 'text' => 'Download Data...', + 'remoteWeb' => true + ], + // [ + // 'method' => 'testProcess', + // 'text' => 'Process Data...', + // 'steps' => true + // ] + ] + ); + + $this->testPackage->testTest(); + $this->testPackage->testTest(); + $this->testPackage->testTest(); + $this->testPackage->testDownload(); + + $this->addResponse( + $this->testPackage->packagesData->responseMessage, + $this->testPackage->packagesData->responseCode + ); + } } \ No newline at end of file diff --git a/apps/Core/Components/System/Progress/ProgressComponent.php b/apps/Core/Components/System/Progress/ProgressComponent.php index 057994c09..3ccb8be57 100644 --- a/apps/Core/Components/System/Progress/ProgressComponent.php +++ b/apps/Core/Components/System/Progress/ProgressComponent.php @@ -33,4 +33,32 @@ public function getProgressAction() ); } } + + public function cancelProgressAction() + { + $this->requestIsPost(); + + $fileName = $this->session->getId(); + + if (isset($this->postData()['session'])) { + $fileName = $this->postData()['session']; + } else if (isset($this->postData()['file_name'])) { + $fileName = $this->postData()['file_name']; + } + + $progress = $this->basepackages->progress->cancelProgress($fileName); + + if ($progress) { + $this->addResponse( + $this->basepackages->progress->packagesData->responseMessage, + $this->basepackages->progress->packagesData->responseCode, + $progress, + ); + } else { + $this->addResponse( + $this->basepackages->progress->packagesData->responseMessage, + $this->basepackages->progress->packagesData->responseCode + ); + } + } } \ No newline at end of file diff --git a/apps/Core/Views/Default/html/devtools/test/view.html b/apps/Core/Views/Default/html/devtools/test/view.html index bd9098bad..304c517cf 100644 --- a/apps/Core/Views/Default/html/devtools/test/view.html +++ b/apps/Core/Views/Default/html/devtools/test/view.html @@ -1,4 +1,53 @@ -Test +
+
+ {{adminltetags.useTag('buttons', + [ + 'component' : component, + 'componentName' : componentName, + 'componentId' : componentId, + 'sectionId' : 'test', + 'buttonType' : 'button', + 'buttons' : + { + 'download-extract' : { + 'title' : 'Process', + 'size' : 'sm', + 'icon' : 'cog fa-spin', + 'iconHidden' : true + } + } + ] + )}} +
+
+ \ No newline at end of file diff --git a/public/core/default/js/footer/core/Baz/BazProgress.js b/public/core/default/js/footer/core/Baz/BazProgress.js index 5b1479cd4..0aefea7e0 100644 --- a/public/core/default/js/footer/core/Baz/BazProgress.js +++ b/public/core/default/js/footer/core/Baz/BazProgress.js @@ -1,5 +1,5 @@ /* exported BazProgress */ -/* globals BazHelpers */ +/* globals BazHelpers paginatedPNotify Swal dataCollection */ /* * @title : BazProgress * @description : Baz Progress Lib @@ -18,9 +18,8 @@ var BazProgress = function() { var initialized = false; var progressCounter = 0; var online = false; - var element, manualShowHide, hasChild, hasRemoteWeb; + var element, manualShowHide, hasChild, hasSubProcess, hasCancelButton; var callableFunc = null; - var dataCollection = window.dataCollection; var url var postData = { }; var progressOptions; @@ -33,6 +32,8 @@ var BazProgress = function() { var isUpload = false; var isDownload = false; var isSteps = false; + var pid = 0; + var progressFile = null; // Error // function error(errorMsg) { // throw new Error(errorMsg); @@ -67,11 +68,12 @@ var BazProgress = function() { console.log('Progress service offline'); } - function buildProgressBar(el, mSH = false, hC = false, hRW = false) { + function buildProgressBar(el, mSH = false, hC = false, hSP = false, hCB = true) { element = el; manualShowHide = mSH; hasChild = hC; - hasRemoteWeb = hRW; + hasSubProcess = hSP; + hasCancelButton = hCB; $(element).html( '
' + @@ -99,7 +101,7 @@ var BazProgress = function() { ); } - if (hasRemoteWeb) { + if (hasSubProcess) { $(element).append( '' ); } + + if (hasCancelButton) { + $(element).append( + '' + ); + + $('#' + $(element)[0].id + '-cancel').off(); + $('#' + $(element)[0].id + '-cancel').click(function() { + if (pid > 0) { + Swal.fire({ + title : ' Cancel current process?', + icon : 'question', + background : 'rgba(0,0,0,.8)', + backdrop : 'rgba(0,0,0,.6)', + buttonsStyling : false, + confirmButtonText : 'Yes', + customClass : { + 'confirmButton' : 'btn btn-danger btn-sm text-uppercase', + 'cancelButton' : 'ml-2 btn btn-secondary btn-sm text-uppercase', + }, + showCancelButton : true, + keydownListenerCapture : true, + allowOutsideClick : true, + allowEscapeKey : true, + didOpen : function() { + dataCollection.env.sounds.swalSound.play(); + } + }).then((result) => { + if (result.value) { + var cancelUrl = + dataCollection.env.httpScheme + '://' + + dataCollection.env.httpHost + '/' + + dataCollection.env.appRoute + '/system/progress/cancelProgress'; + + var postData = { }; + postData[$('#security-token').attr('name')] = $('#security-token').val(); + + if (progressFile) { + postData['file_name'] = progressFile; + } + + $.post(cancelUrl, postData, function(response) { + if (response.responseCode == 0) { + paginatedPNotify('success', { + 'title' : response.responseMessage + }); + + resetProgressCounter(); + } else { + paginatedPNotify('error', { + 'title' : response.responseMessage + }); + } + }, 'json'); + } + }); + } + }); + } } function getProgress(options) { @@ -167,6 +232,15 @@ var BazProgress = function() { responseData = response.responseData; } + if (responseData['pid']) { + $('#' + $(element)[0].id + '-cancel').attr('hidden', false); + pid = responseData['pid']; + } + + if (responseData['progressFile']) { + progressFile = responseData['progressFile']; + } + if (responseData['preCheckComplete'] == false || (responseData['callResult'] && responseData['callResult'] === 'reset') ) { @@ -202,6 +276,14 @@ var BazProgress = function() { $('.' + $(element)[0].id + '-remote-bar').css('width', responseData['percentComplete'] + '%'); $('.' + $(element)[0].id + '-remote-bar').attr('aria-valuenow', responseData['percentComplete']); + + if ($('.' + $(element)[0].id + '-progress-span').html() === '') { + $('.' + $(element)[0].id + '-progress-span') + .html(responseData['runners']['running']['text'] + ' (' + responseData['totalPercentComplete'] + '%)'); + + $('.' + $(element)[0].id + '-bar').css('width', responseData['totalPercentComplete'] + '%'); + $('.' + $(element)[0].id + '-bar').attr('aria-valuenow', responseData['totalPercentComplete']); + } } else { $('.progress-remote, .remote-progress-span').attr('hidden', true); @@ -236,6 +318,14 @@ var BazProgress = function() { $('.' + $(element)[0].id + '-remote-bar').css('width', responseData['percentComplete'] + '%'); $('.' + $(element)[0].id + '-remote-bar').attr('aria-valuenow', responseData['percentComplete']); + + if ($('.' + $(element)[0].id + '-progress-span').html() === '') { + $('.' + $(element)[0].id + '-progress-span') + .html(responseData['runners']['running']['text'] + ' (' + responseData['totalPercentComplete'] + '%)'); + + $('.' + $(element)[0].id + '-bar').css('width', responseData['totalPercentComplete'] + '%'); + $('.' + $(element)[0].id + '-bar').attr('aria-valuenow', responseData['totalPercentComplete']); + } } else { $('.' + $(element)[0].id + '-remote-bar').css('width', '0%'); $('.' + $(element)[0].id + '-remote-bar').attr('aria-valuenow', 0); @@ -245,10 +335,10 @@ var BazProgress = function() { $('.progress-remote, .remote-progress-span').attr('hidden', true); $('.' + $(element)[0].id + '-progress-span') - .html(responseData['runners']['running']['text'] + ' (' + responseData['percentComplete'] + '%)'); + .html(responseData['runners']['running']['text'] + ' (' + responseData['totalPercentComplete'] + '%)'); - $('.' + $(element)[0].id + '-bar').css('width', responseData['percentComplete'] + '%'); - $('.' + $(element)[0].id + '-bar').attr('aria-valuenow', responseData['percentComplete']); + $('.' + $(element)[0].id + '-bar').css('width', responseData['totalPercentComplete'] + '%'); + $('.' + $(element)[0].id + '-bar').attr('aria-valuenow', responseData['totalPercentComplete']); } } } else { @@ -305,6 +395,8 @@ var BazProgress = function() { isUpload = false; isDownload = false; isSteps = false; + pid = 0; + progressFile = null; $('.' + $(element)[0].id + '-child-bar').css('width', '0%'); $('.' + $(element)[0].id + '-child-bar').attr('aria-valuenow', 0); switchProgressBarColor('.' + $(element)[0].id + '-child-bar', 'info'); @@ -312,6 +404,7 @@ var BazProgress = function() { $('.' + $(element)[0].id + '-remote-bar').attr('aria-valuenow', 0); switchProgressBarColor('.' + $(element)[0].id + '-remote-bar', 'info'); $('.progress-remote, .remote-progress-span').attr('hidden', true); + $('#' + $(element)[0].id + '-cancel').attr('hidden', true); $('body').trigger('bazProgressComplete'); } } else { @@ -410,6 +503,8 @@ var BazProgress = function() { isUpload = false; isDownload = false; isSteps = false; + pid = 0; + progressFile = null; $('body').trigger({'type':'bazProgressComplete', 'reset' : true}); } @@ -448,12 +543,15 @@ var BazProgress = function() { getProgress(options); } } - BazProgress.buildProgressBar = function(el, mSH = false, child = false, remoteWeb = false) { - buildProgressBar(el, mSH, child, remoteWeb); + BazProgress.buildProgressBar = function(el, mSH = false, child = false, hasSubProcess = false, hasCancelButton = true) { + buildProgressBar(el, mSH, child, hasSubProcess, hasCancelButton); } BazProgress.switchProgressBarColor = function(el, color) { switchProgressBarColor(el, color); } + BazProgress.resetProgressCounter = function() { + resetProgressCounter(); + } BazProgress.setCallable = function(callable) { setCallable(callable); } @@ -465,4 +563,4 @@ var BazProgress = function() { setup(bazProgressConstructor); return bazProgressConstructor; -}(); \ No newline at end of file +}(); diff --git a/public/core/default/js/footer/jsFooterCore.js b/public/core/default/js/footer/jsFooterCore.js index 59e736cb4..6ad89299b 100644 --- a/public/core/default/js/footer/jsFooterCore.js +++ b/public/core/default/js/footer/jsFooterCore.js @@ -11826,7 +11826,7 @@ var BazAnnouncements = function() { return bazAnnouncementsConstructor; }(); /* exported BazProgress */ -/* globals BazHelpers */ +/* globals BazHelpers paginatedPNotify Swal dataCollection */ /* * @title : BazProgress * @description : Baz Progress Lib @@ -11845,9 +11845,8 @@ var BazProgress = function() { var initialized = false; var progressCounter = 0; var online = false; - var element, manualShowHide, hasChild, hasRemoteWeb; + var element, manualShowHide, hasChild, hasSubProcess, hasCancelButton; var callableFunc = null; - var dataCollection = window.dataCollection; var url var postData = { }; var progressOptions; @@ -11860,6 +11859,8 @@ var BazProgress = function() { var isUpload = false; var isDownload = false; var isSteps = false; + var pid = 0; + var progressFile = null; // Error // function error(errorMsg) { // throw new Error(errorMsg); @@ -11894,11 +11895,12 @@ var BazProgress = function() { console.log('Progress service offline'); } - function buildProgressBar(el, mSH = false, hC = false, hRW = false) { + function buildProgressBar(el, mSH = false, hC = false, hSP = false, hCB = true) { element = el; manualShowHide = mSH; hasChild = hC; - hasRemoteWeb = hRW; + hasSubProcess = hSP; + hasCancelButton = hCB; $(element).html( '
' + @@ -11926,7 +11928,7 @@ var BazProgress = function() { ); } - if (hasRemoteWeb) { + if (hasSubProcess) { $(element).append( '' ); } + + if (hasCancelButton) { + $(element).append( + '' + ); + + $('#' + $(element)[0].id + '-cancel').off(); + $('#' + $(element)[0].id + '-cancel').click(function() { + if (pid > 0) { + Swal.fire({ + title : ' Cancel current process?', + icon : 'question', + background : 'rgba(0,0,0,.8)', + backdrop : 'rgba(0,0,0,.6)', + buttonsStyling : false, + confirmButtonText : 'Yes', + customClass : { + 'confirmButton' : 'btn btn-danger btn-sm text-uppercase', + 'cancelButton' : 'ml-2 btn btn-secondary btn-sm text-uppercase', + }, + showCancelButton : true, + keydownListenerCapture : true, + allowOutsideClick : true, + allowEscapeKey : true, + didOpen : function() { + dataCollection.env.sounds.swalSound.play(); + } + }).then((result) => { + if (result.value) { + var cancelUrl = + dataCollection.env.httpScheme + '://' + + dataCollection.env.httpHost + '/' + + dataCollection.env.appRoute + '/system/progress/cancelProgress'; + + var postData = { }; + postData[$('#security-token').attr('name')] = $('#security-token').val(); + + if (progressFile) { + postData['file_name'] = progressFile; + } + + $.post(cancelUrl, postData, function(response) { + if (response.responseCode == 0) { + paginatedPNotify('success', { + 'title' : response.responseMessage + }); + + resetProgressCounter(); + } else { + paginatedPNotify('error', { + 'title' : response.responseMessage + }); + } + }, 'json'); + } + }); + } + }); + } } function getProgress(options) { @@ -11994,6 +12059,15 @@ var BazProgress = function() { responseData = response.responseData; } + if (responseData['pid']) { + $('#' + $(element)[0].id + '-cancel').attr('hidden', false); + pid = responseData['pid']; + } + + if (responseData['progressFile']) { + progressFile = responseData['progressFile']; + } + if (responseData['preCheckComplete'] == false || (responseData['callResult'] && responseData['callResult'] === 'reset') ) { @@ -12029,6 +12103,14 @@ var BazProgress = function() { $('.' + $(element)[0].id + '-remote-bar').css('width', responseData['percentComplete'] + '%'); $('.' + $(element)[0].id + '-remote-bar').attr('aria-valuenow', responseData['percentComplete']); + + if ($('.' + $(element)[0].id + '-progress-span').html() === '') { + $('.' + $(element)[0].id + '-progress-span') + .html(responseData['runners']['running']['text'] + ' (' + responseData['totalPercentComplete'] + '%)'); + + $('.' + $(element)[0].id + '-bar').css('width', responseData['totalPercentComplete'] + '%'); + $('.' + $(element)[0].id + '-bar').attr('aria-valuenow', responseData['totalPercentComplete']); + } } else { $('.progress-remote, .remote-progress-span').attr('hidden', true); @@ -12063,6 +12145,14 @@ var BazProgress = function() { $('.' + $(element)[0].id + '-remote-bar').css('width', responseData['percentComplete'] + '%'); $('.' + $(element)[0].id + '-remote-bar').attr('aria-valuenow', responseData['percentComplete']); + + if ($('.' + $(element)[0].id + '-progress-span').html() === '') { + $('.' + $(element)[0].id + '-progress-span') + .html(responseData['runners']['running']['text'] + ' (' + responseData['totalPercentComplete'] + '%)'); + + $('.' + $(element)[0].id + '-bar').css('width', responseData['totalPercentComplete'] + '%'); + $('.' + $(element)[0].id + '-bar').attr('aria-valuenow', responseData['totalPercentComplete']); + } } else { $('.' + $(element)[0].id + '-remote-bar').css('width', '0%'); $('.' + $(element)[0].id + '-remote-bar').attr('aria-valuenow', 0); @@ -12072,10 +12162,10 @@ var BazProgress = function() { $('.progress-remote, .remote-progress-span').attr('hidden', true); $('.' + $(element)[0].id + '-progress-span') - .html(responseData['runners']['running']['text'] + ' (' + responseData['percentComplete'] + '%)'); + .html(responseData['runners']['running']['text'] + ' (' + responseData['totalPercentComplete'] + '%)'); - $('.' + $(element)[0].id + '-bar').css('width', responseData['percentComplete'] + '%'); - $('.' + $(element)[0].id + '-bar').attr('aria-valuenow', responseData['percentComplete']); + $('.' + $(element)[0].id + '-bar').css('width', responseData['totalPercentComplete'] + '%'); + $('.' + $(element)[0].id + '-bar').attr('aria-valuenow', responseData['totalPercentComplete']); } } } else { @@ -12132,6 +12222,8 @@ var BazProgress = function() { isUpload = false; isDownload = false; isSteps = false; + pid = 0; + progressFile = null; $('.' + $(element)[0].id + '-child-bar').css('width', '0%'); $('.' + $(element)[0].id + '-child-bar').attr('aria-valuenow', 0); switchProgressBarColor('.' + $(element)[0].id + '-child-bar', 'info'); @@ -12139,6 +12231,7 @@ var BazProgress = function() { $('.' + $(element)[0].id + '-remote-bar').attr('aria-valuenow', 0); switchProgressBarColor('.' + $(element)[0].id + '-remote-bar', 'info'); $('.progress-remote, .remote-progress-span').attr('hidden', true); + $('#' + $(element)[0].id + '-cancel').attr('hidden', true); $('body').trigger('bazProgressComplete'); } } else { @@ -12237,6 +12330,8 @@ var BazProgress = function() { isUpload = false; isDownload = false; isSteps = false; + pid = 0; + progressFile = null; $('body').trigger({'type':'bazProgressComplete', 'reset' : true}); } @@ -12275,12 +12370,15 @@ var BazProgress = function() { getProgress(options); } } - BazProgress.buildProgressBar = function(el, mSH = false, child = false, remoteWeb = false) { - buildProgressBar(el, mSH, child, remoteWeb); + BazProgress.buildProgressBar = function(el, mSH = false, child = false, hasSubProcess = false, hasCancelButton = true) { + buildProgressBar(el, mSH, child, hasSubProcess, hasCancelButton); } BazProgress.switchProgressBarColor = function(el, color) { switchProgressBarColor(el, color); } + BazProgress.resetProgressCounter = function() { + resetProgressCounter(); + } BazProgress.setCallable = function(callable) { setCallable(callable); } @@ -12292,4 +12390,4 @@ var BazProgress = function() { setup(bazProgressConstructor); return bazProgressConstructor; -}(); \ No newline at end of file +}(); diff --git a/system/Base/Providers/BasepackagesServiceProvider/Packages/Progress.php b/system/Base/Providers/BasepackagesServiceProvider/Packages/Progress.php index d2eeb6703..2cd7b9e6e 100644 --- a/system/Base/Providers/BasepackagesServiceProvider/Packages/Progress.php +++ b/system/Base/Providers/BasepackagesServiceProvider/Packages/Progress.php @@ -8,6 +8,7 @@ use League\Flysystem\UnableToWriteFile; use Mattiasgeniar\Percentage\Percentage; use System\Base\BasePackage; +use System\Base\Exceptions\DuplicateProgressException; class Progress extends BasePackage { @@ -44,8 +45,9 @@ public function checkProgressFile($fileName = null) $this->progressFileName = $fileName; } - if ($this->readProgressFile()) { - return true; + $progressFile = $this->readProgressFile(); + if ($progressFile) { + return $progressFile; } return false; @@ -69,6 +71,8 @@ public function registerMethods(array $methods) $progressFile = $this->readProgressFile(); + $this->checkProcessIsRunning($progressFile); + if ($progressFile && isset($progressFile['processes']) && count($progressFile['processes']) > 0) { foreach ($methods as $method) { array_push($progressFile['processes'], $method); @@ -132,11 +136,12 @@ public function getProgress($session = null, $returnArray = false) $progress = [ - 'total' => $progressFile['total'], - 'completed' => $progressFile['completed'], - 'preCheckComplete' => $progressFile['preCheckComplete'], - 'percentComplete' => $this->getPercentComplete($progressFile), - 'runners' => $progressFile['runners'] ?? false + 'total' => $progressFile['total'], + 'completed' => $progressFile['completed'], + 'preCheckComplete' => $progressFile['preCheckComplete'], + 'totalPercentComplete' => $this->getPercentComplete($progressFile, false), + 'percentComplete' => $this->getPercentComplete($progressFile), + 'runners' => $progressFile['runners'] ?? false ]; if ($returnArray) { @@ -170,6 +175,13 @@ public function updateProgress($method, $callResult = null, $deleteFile = true, $progressFile = $this->readProgressFile(); if (isset($progressFile['processes']) && count($progressFile['processes']) > 0) { + if ($progressFile['allProcesses'][0]['method'] === $method && + !$callResult && + $progressFile['completed'] === 0 + ) { + $this->checkProcessIsRunning($progressFile); + } + $runners = []; foreach ($progressFile['processes'] as $progressFileKey => $progressFileMethod) { @@ -250,6 +262,32 @@ public function updateProgress($method, $callResult = null, $deleteFile = true, return false; } + public function checkProcessIsRunning($progressFile = null) + { + if (!$progressFile) { + $progressFile = $this->checkProgressFile(); + } + + //Check if process is already running + //If a user tries to reinitiate same process from start, we should return an error. + if ($progressFile && + ($progressFile['runners']['running'] !== false) && + $progressFile['pid'] > 0 + ) { + if (isset($progressFile['pid']) && $progressFile['pid'] > 0) { + exec('ps -aux | grep ' . $progressFile['pid'], $output, $result); + + if ($result === 0 && + count($output) > 0 + ) { + if (str_contains($output[0], 'php')) { + throw new DuplicateProgressException('Process is already running with process ID: ' . $progressFile['pid']); + } + } + } + } + } + protected function checkNotificationTunnel() { if (!$this->notificationsTunnel && isset($this->apps)) { @@ -295,12 +333,15 @@ protected function sendNotification($callResult, $counters = null) 'responseMessage' => 'Ok', 'responseData' => [ - 'total' => $progressFile['total'], - 'completed' => $progressFile['completed'], - 'preCheckComplete' => $progressFile['preCheckComplete'], - 'percentComplete' => $this->getPercentComplete($progressFile), - 'runners' => $progressFile['runners'] ?? false, - 'callResult' => $callResult + 'progressFile' => $this->progressFileName, + 'pid' => $progressFile['pid'] ?? false, + 'total' => $progressFile['total'], + 'completed' => $progressFile['completed'], + 'preCheckComplete' => $progressFile['preCheckComplete'], + 'totalPercentComplete' => $this->getPercentComplete($progressFile, false), + 'percentComplete' => $this->getPercentComplete($progressFile), + 'runners' => $progressFile['runners'] ?? false, + 'callResult' => $callResult ] ] ] @@ -309,31 +350,33 @@ protected function sendNotification($callResult, $counters = null) } } - protected function getPercentComplete($progressFile) + protected function getPercentComplete($progressFile, $counters = true) { $percentComplete = (float) number_format(($progressFile['completed'] * 100) / $progressFile['total']); - if (isset($progressFile['runners']['running']['remoteWebCounters'])) { - $webProgress = 0; + if ($counters) { + if (isset($progressFile['runners']['running']['remoteWebCounters'])) { + $webProgress = 0; - if (isset($progressFile['runners']['running']['remoteWebCounters']['downloadTotal']) && $progressFile['runners']['running']['remoteWebCounters']['downloadTotal'] > 0) { - $webProgress = Percentage::calculate($progressFile['runners']['running']['remoteWebCounters']['downloadedBytes'], $progressFile['runners']['running']['remoteWebCounters']['downloadTotal']); - } else if (isset($progressFile['runners']['running']['remoteWebCounters']['uploadTotal']) && $progressFile['runners']['running']['remoteWebCounters']['uploadTotal'] > 0) { - $webProgress = Percentage::calculate($progressFile['runners']['running']['remoteWebCounters']['uploadedBytes'], $progressFile['runners']['running']['remoteWebCounters']['uploadTotal']); - } + if (isset($progressFile['runners']['running']['remoteWebCounters']['downloadTotal']) && $progressFile['runners']['running']['remoteWebCounters']['downloadTotal'] > 0) { + $webProgress = Percentage::calculate($progressFile['runners']['running']['remoteWebCounters']['downloadedBytes'], $progressFile['runners']['running']['remoteWebCounters']['downloadTotal']); + } else if (isset($progressFile['runners']['running']['remoteWebCounters']['uploadTotal']) && $progressFile['runners']['running']['remoteWebCounters']['uploadTotal'] > 0) { + $webProgress = Percentage::calculate($progressFile['runners']['running']['remoteWebCounters']['uploadedBytes'], $progressFile['runners']['running']['remoteWebCounters']['uploadTotal']); + } - if ($webProgress > -1) { - $percentComplete = (float) number_format($webProgress); - } - } else if (isset($progressFile['runners']['running']['stepsCounters'])) { - $stepsProgress = 0; + if ($webProgress > -1) { + $percentComplete = (float) number_format($webProgress); + } + } else if (isset($progressFile['runners']['running']['stepsCounters'])) { + $stepsProgress = 0; - if (isset($progressFile['runners']['running']['stepsCounters']['stepsTotal']) && $progressFile['runners']['running']['stepsCounters']['stepsTotal'] > 0) { - $stepsProgress = Percentage::calculate($progressFile['runners']['running']['stepsCounters']['stepsCurrent'], $progressFile['runners']['running']['stepsCounters']['stepsTotal']); - } + if (isset($progressFile['runners']['running']['stepsCounters']['stepsTotal']) && $progressFile['runners']['running']['stepsCounters']['stepsTotal'] > 0) { + $stepsProgress = Percentage::calculate($progressFile['runners']['running']['stepsCounters']['stepsCurrent'], $progressFile['runners']['running']['stepsCounters']['stepsTotal']); + } - if ($stepsProgress > -1) { - $percentComplete = (float) number_format($stepsProgress); + if ($stepsProgress > -1) { + $percentComplete = (float) number_format($stepsProgress); + } } } @@ -379,6 +422,49 @@ protected function checkProgressPath() return true; } + public function cancelProgress($fileName = null) + { + $progressFile = $this->checkProgressFile($fileName); + + if ($progressFile) { + if (isset($progressFile['pid']) && $progressFile['pid'] > 0) { + exec('ps -aux | grep ' . $progressFile['pid'], $output, $result); + + if ($result === 0 && + count($output) > 0 + ) { + if (str_contains($output[0], 'php')) { + exec('kill -9 ' . $progressFile['pid'], $output, $result); + + if ($result !== 0) { + $this->addResponse('Error terminating process', 1, ['output' => $output]); + + return false; + } + + $this->writeProgressFile(methods: [],progressFile: $progressFile); + + $this->addResponse('Successfully terminating process'); + + return $this->resetProgress(); + } + } + } + + $progressFile['runners']['running'] = false; + $progressFile['runners']['next'] = false; + $this->writeProgressFile(methods: [], progressFile: $progressFile, register: true); + + $this->addResponse('Process not running', 1); + + return false; + } else { + $this->addResponse('Error loading progressfile!', 1); + + return false; + } + } + protected function readProgressFile($session = null) { if (!$this->progressFileName) { @@ -496,6 +582,14 @@ protected function writeProgressFile( $file['notifications_tunnel'] = $this->notificationsTunnel; + if ($register) { + $file['pid'] = 0; + } + + if (!isset($file['pid'])) { + $file['pid'] = getmypid(); + } + if ($this->opCache) { if ($progressFile) { $this->opCache->resetCache($this->progressFileName, $file, 'progress'); @@ -513,6 +607,8 @@ protected function writeProgressFile( public function deleteProgressFile() { + $this->checkProcessIsRunning(); + if (!$this->progressFileName) { $this->progressFileName = $this->session->getId(); } diff --git a/system/Base/Providers/DispatcherServiceProvider/Dispatcher.php b/system/Base/Providers/DispatcherServiceProvider/Dispatcher.php index 759817c53..d8f4893e2 100644 --- a/system/Base/Providers/DispatcherServiceProvider/Dispatcher.php +++ b/system/Base/Providers/DispatcherServiceProvider/Dispatcher.php @@ -76,6 +76,10 @@ function ( $dispatcher->getDi()->getShared('logger')->logExceptions->critical(json_trace($exception)); } + if ($dispatcher->getDi()->getShared('request')->isPost()) { + throw $exception; + } + if (!str_contains($exception->getMessage(), 'handler class cannot be loaded')) {//Handle any other exceptions, like variable not found, etc $class = (new \ReflectionClass($exception))->getShortName(); diff --git a/system/Base/Providers/ErrorServiceProvider/Error.php b/system/Base/Providers/ErrorServiceProvider/Error.php index e8a56b2b1..f5fba2b38 100644 --- a/system/Base/Providers/ErrorServiceProvider/Error.php +++ b/system/Base/Providers/ErrorServiceProvider/Error.php @@ -93,6 +93,10 @@ public function handle(\throwable $exception) $customHandler = $this->customHandler(); + if ($this->logger) { + $this->logger->commit(); + } + if (!$customHandler) { if ($this->appDebug) { $this->showOnScreen(); diff --git a/system/Base/Providers/ErrorServiceProvider/ExceptionHandlers.php b/system/Base/Providers/ErrorServiceProvider/ExceptionHandlers.php index 5ae5ac452..863c97aef 100644 --- a/system/Base/Providers/ErrorServiceProvider/ExceptionHandlers.php +++ b/system/Base/Providers/ErrorServiceProvider/ExceptionHandlers.php @@ -95,6 +95,13 @@ public function handleIOException($exception) return $this->sendJson(); } + public function handleDuplicateProgressException($exception) + { + $this->addResponse($exception->getMessage(), 1); + + return $this->sendJson(); + } + private function setViewsDir($partial) { $this->view->setViewsDir(base_path($this->baseErrorDir)); From 07379a55e6ac547a9bea25a64e798e73c97da1f1 Mon Sep 17 00:00:00 2001 From: "Gurdeep Singh (Guru)" Date: Tue, 25 Feb 2025 19:44:53 +1100 Subject: [PATCH 10/11] update #628 --- .../Packages/Devtools/Test/DevtoolsTest.php | 356 ++++++++++++++++++ .../Exceptions/DuplicateProgressException.php | 7 + 2 files changed, 363 insertions(+) create mode 100644 apps/Core/Packages/Devtools/Test/DevtoolsTest.php create mode 100644 system/Base/Exceptions/DuplicateProgressException.php diff --git a/apps/Core/Packages/Devtools/Test/DevtoolsTest.php b/apps/Core/Packages/Devtools/Test/DevtoolsTest.php new file mode 100644 index 000000000..2175d45f3 --- /dev/null +++ b/apps/Core/Packages/Devtools/Test/DevtoolsTest.php @@ -0,0 +1,356 @@ +sourceDir))) { + if (!mkdir(base_path($this->sourceDir), 0777, true)) { + return false; + } + } + + //Increase Exectimeout to 3 hours as this process takes time to extract and merge data. + if ((int) ini_get('max_execution_time') < 10800) { + set_time_limit(10800); + } + + //Increase memory_limit to 2G as the process takes a bit of memory to process the array. + if ((int) ini_get('memory_limit') < 1024) { + ini_set('memory_limit', '1024M'); + } + + $this->now = \Carbon\Carbon::now(); + + parent::onConstruct(); + } + + public function __call($method, $arguments) + { + if (method_exists($this, $method)) { + $this->basepackages->progress->updateProgress($method, null, false); + + $call = call_user_func_array([$this, $method], $arguments); + + $callResult = $call; + + if ($call !== false) { + $call = true; + } + + $this->basepackages->progress->updateProgress($method, $call, false); + + return $callResult; + } + } + + protected function testTest() + { + sleep(1); + + return true; + } + + protected function testDownload() + { + $today = $this->now->toDateString(); + + try { + //File is already extracted + if ($this->localContent->fileExists($this->sourceDir . $today . '-funds.db')) { + return true; + } + + //File is already downloaded + if ($this->localContent->fileExists($this->sourceDir . $today . '-funds.db.zst')) { + return true; + } + } catch (FilesystemException | UnableToCheckExistence | UnableToRetrieveMetadata | \throwable $e) { + $this->addResponse($e->getMessage(), 1); + + return false; + } + + $this->method = 'testDownload'; + + return $this->downloadData($this->sourceLink, base_path($this->sourceDir) . $today . '-funds.db.zst'); + } + + protected function downloadData($url, $sink) + { + $download = $this->remoteWebContent->request( + 'GET', + $url, + [ + 'progress' => function( + $downloadTotal, + $downloadedBytes, + $uploadTotal, + $uploadedBytes + ) { + $counters = + [ + 'downloadTotal' => $downloadTotal, + 'downloadedBytes' => $downloadedBytes, + 'uploadTotal' => $uploadTotal, + 'uploadedBytes' => $uploadedBytes + ]; + + if ($downloadedBytes === 0) { + return; + } + + //Trackcounter is needed as guzzelhttp runs this in a while loop causing too many updates with same download count. + //So this way, we only update progress when there is actually an update. + if ($downloadedBytes === $this->trackCounter) { + return; + } + + $this->trackCounter = $downloadedBytes; + + if ($downloadedBytes === $downloadTotal) { + $this->basepackages->progress->updateProgress($this->method, true, false, null, $counters); + } else { + $this->basepackages->progress->updateProgress($this->method, null, false, null, $counters); + } + }, + 'verify' => false, + 'connect_timeout' => 5, + 'sink' => $sink + ] + ); + + $this->trackCounter = 0; + + if ($download->getStatusCode() === 200) { + return true; + } + + $this->addResponse('Download resulted in : ' . $download->getStatusCode(), 1); + + return false; + } + + protected function testProcess() + { + $this->method = 'testProcess'; + + $today = $this->now->toDateString(); + + try { + //File is already extracted + if (!$this->localContent->fileExists($this->sourceDir . $today . '-funds.db')) { + $this->addResponse('File not downloaded and extracted correctly!', 1); + + return false; + } + } catch (FilesystemException | UnableToCheckExistence | \throwable $e) { + $this->addResponse($e->getMessage(), 1); + + return false; + } + + try { + $sqlite = (new Sqlite())->init(base_path($this->sourceDir . $today . '-funds.db')); + } catch (\throwable $e) { + $this->addResponse('Unable to open database file', 1); + + return false; + } + + //Test + // try { + // $this->basepackages->utils->setMicroTimer('DBStart', true); + + // $isinNavs = + // $sqlite->query( + // "SELECT * from nav N + // JOIN securities S ON N.scheme_code = S.scheme_code + // WHERE S.isin = 'INF760K01FV4' + // AND N.date >= '2000-01-01' + // ORDER BY N.date DESC" + // )->fetchAll(Enum::FETCH_ASSOC); + + // $this->basepackages->utils->setMicroTimer('DBStop', true); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: false, dumpTraces: false); + + // $this->basepackages->utils->resetMicroTimer(); + + // $this->basepackages->utils->setMicroTimer('FFReadStart', true); + // $this->navsPackage = new MfNavs; + // $new = $this->navsPackage->getMfNavsByIsin('INF760K01FV4'); + // $this->basepackages->utils->setMicroTimer('FFReadStop', true); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: false, dumpTraces: false); + + // $this->basepackages->utils->resetMicroTimer(); + + // $this->basepackages->utils->setMicroTimer('FFReadStart2', true); + // $this->navsPackage = new MfNavs; + // $new = $this->navsPackage->getById(1); + // $this->basepackages->utils->setMicroTimer('FFReadStop2', true); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: false, dumpTraces: false); + // die(); + // } catch (\throwable $e) { + // trace([$e]); + // } + //Test + + try { + $this->navsPackage = new MfNavs; + + $isins = $sqlite->query("SELECT * from securities")->fetchAll(Enum::FETCH_ASSOC); + + if ($isins && count($isins) > 0) { + $isinsTotal = 1000; + foreach ($isins as $key => $isin) { + $this->basepackages->utils->setMicroTimer('Start'); + $dbIsin = $this->navsPackage->getMfNavsByIsin($isin['isin']); + + // $lastUpdated = $this->now->subDay(1)->toDateString(); + $lastUpdated = '2000-01-01'; + + if (!$dbIsin) { + $dbIsin = []; + $dbIsin['type'] = $isin['type']; + $dbIsin['scheme_code'] = $isin['scheme_code']; + $dbIsin['isin'] = $isin['isin']; + $dbIsin['navs'] = []; + } else { + $lastUpdated = $dbIsin['last_updated']; + } + + $isin = $isin['isin']; + // $this->basepackages->utils->setMicroTimer('DBStart', true); + $isinNavs = + $sqlite->query( + "SELECT * from nav N + JOIN securities S ON N.scheme_code = S.scheme_code + WHERE S.isin = '$isin' + AND N.date >= '$lastUpdated' + ORDER BY N.date ASC" + )->fetchAll(Enum::FETCH_ASSOC); + + // $this->basepackages->utils->setMicroTimer('DBStop', true); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: false, dumpTraces: false); + if ($isinNavs && count($isinNavs) > 0) { + $dbIsin['last_updated'] = $this->helper->last($isinNavs)['date']; + $dbIsin['latest_nav'] = $this->helper->last($isinNavs)['nav']; + foreach ($isinNavs as $isinNav) { + $dbIsin['navs'][$isinNav['date']] = $isinNav['nav']; + } + } + + if (isset($dbIsin['id'])) { + $this->navsPackage->update($dbIsin); + } else { + $this->navsPackage->addMfNavs($dbIsin); + } + // $this->basepackages->utils->setMicroTimer('FFStop', true); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: false, dumpTraces: false); + + // $this->basepackages->utils->setMicroTimer('FFReadStart', true); + // $new = $this->navsPackage->getMfNavsByIsin($isin); + $this->basepackages->utils->setMicroTimer('End'); + + $time = $this->basepackages->utils->getMicroTimer(); + + if ($time && isset($time[1]['difference']) && $time[1]['difference'] !== 0) { + $totalTime = date("H:i:s", floor($time[1]['difference'] * ($isinsTotal - $key))); + // $this->basepackages->utils->formatMicrotime($time[1]['difference'] * $isinsTotal); + } + + $this->basepackages->utils->resetMicroTimer(); + // trace(varsToDump : [$this->basepackages->utils->getMicroTimer()], exit: true, dumpTraces: false); + $this->basepackages->progress->updateProgress( + method: $this->method, + counters: ['stepsTotal' => $isinsTotal, 'stepsCurrent' => ($key + 1)], + text: 'Time remaining : ' . $totalTime . '...' + ); + if ($key === 1000) { + return true; + } + } + } + } catch (\throwable $e) { + trace([$e]); + } + + return true; + // $statement = $sqlite->prepare('SELECT * from nav LIMIT 0,1'); + // trace([$sqlite->query('SELECT * from securities LIMIT 0,10')->fetchAll(Enum::FETCH_ASSOC)]); + // trace([$sqlite->query("SELECT * from nav N JOIN securities S ON N.scheme_code = S.scheme_code WHERE S.isin = 'INF209KB1ZL4' LIMIT 0,10")->fetchAll(Enum::FETCH_ASSOC)]); + // trace([$sqlite->query("SELECT * from nav N JOIN securities S ON N.scheme_code = S.scheme_code WHERE S.isin = 'INF209KB1ZL4'")->fetchAll(Enum::FETCH_ASSOC)]); + } + + // public function getDevtoolsTestById($id) + // { + // $devtoolstest = $this->getById($id); + + // if ($devtoolstest) { + // // + // $this->addResponse('Success'); + + // return; + // } + + // $this->addResponse('Error', 1); + // } + + // public function addDevtoolsTest($data) + // { + // // + // } + + // public function updateDevtoolsTest($data) + // { + // $devtoolstest = $this->getById($id); + + // if ($devtoolstest) { + // // + // $this->addResponse('Success'); + + // return; + // } + + // $this->addResponse('Error', 1); + // } + + // public function removeDevtoolsTest($data) + // { + // $devtoolstest = $this->getById($id); + + // if ($devtoolstest) { + // // + // $this->addResponse('Success'); + + // return; + // } + + // $this->addResponse('Error', 1); + // } +} \ No newline at end of file diff --git a/system/Base/Exceptions/DuplicateProgressException.php b/system/Base/Exceptions/DuplicateProgressException.php new file mode 100644 index 000000000..d9e582db5 --- /dev/null +++ b/system/Base/Exceptions/DuplicateProgressException.php @@ -0,0 +1,7 @@ + Date: Tue, 25 Feb 2025 19:45:23 +1100 Subject: [PATCH 11/11] fix issue #638 --- .../Test/Install/Schema/DevtoolsTest.php | 69 +++++++++++++++++++ .../Devtools/Test/Install/package.json | 39 +++++++++++ .../Devtools/Test/Model/DevtoolsTest.php | 14 ++++ .../Base/Installer/Packages/Setup/Schema.php | 4 ++ 4 files changed, 126 insertions(+) create mode 100644 apps/Core/Packages/Devtools/Test/Install/Schema/DevtoolsTest.php create mode 100644 apps/Core/Packages/Devtools/Test/Install/package.json create mode 100644 apps/Core/Packages/Devtools/Test/Model/DevtoolsTest.php diff --git a/apps/Core/Packages/Devtools/Test/Install/Schema/DevtoolsTest.php b/apps/Core/Packages/Devtools/Test/Install/Schema/DevtoolsTest.php new file mode 100644 index 000000000..11a1ad91c --- /dev/null +++ b/apps/Core/Packages/Devtools/Test/Install/Schema/DevtoolsTest.php @@ -0,0 +1,69 @@ + [ + new Column( + 'id', + [ + 'type' => Column::TYPE_INTEGER, + 'notNull' => true, + 'autoIncrement' => true, + 'primary' => true, + ] + ), + new Column( + 'first_name', + [ + 'type' => Column::TYPE_VARCHAR, + 'size' => 50, + 'notNull' => true, + ] + ), + new Column( + 'last_name', + [ + 'type' => Column::TYPE_VARCHAR, + 'size' => 50, + 'notNull' => true, + ] + ) + ], + 'indexes' => [ + new Index( + 'column_UNIQUE', + [ + 'last_name' + ], + 'UNIQUE' + ) + ], + 'options' => [ + 'TABLE_COLLATION' => 'utf8mb4_general_ci' + ] + ]; + } + + public function indexes() + { + return + [ + new Index( + 'column_INDEX', + [ + 'first_name' + ], + 'INDEX' + ) + ]; + } +} diff --git a/apps/Core/Packages/Devtools/Test/Install/package.json b/apps/Core/Packages/Devtools/Test/Install/package.json new file mode 100644 index 000000000..ccec11df4 --- /dev/null +++ b/apps/Core/Packages/Devtools/Test/Install/package.json @@ -0,0 +1,39 @@ +{ + "name" : "DevtoolsTest", + "display_name" : "Test", + "description" : "

Test package for devtools

", + "module_type" : "packages", + "app_type" : "core", + "category" : "devtools", + "version" : "0.0.0", + "repo" : "https://.../", + "class" : "Apps\\Core\\Packages\\Devtools\\Test\\DevtoolsTest", + "dependencies" : + { + "core" : + { + "name" : "Core", + "version" : "0.0.0", + "repo" : "https://.../" + }, + "apptype" : [], + "packages" : [], + "middlewares" : [], + "externals" : + { + "composer" : + { + "require" : [] + }, + "config" : + { + "allow-plugins" : [] + }, + "extra" : + { + "patches" : [] + } + } + }, + "settings" : [] +} \ No newline at end of file diff --git a/apps/Core/Packages/Devtools/Test/Model/DevtoolsTest.php b/apps/Core/Packages/Devtools/Test/Model/DevtoolsTest.php new file mode 100644 index 000000000..bb04fadcf --- /dev/null +++ b/apps/Core/Packages/Devtools/Test/Model/DevtoolsTest.php @@ -0,0 +1,14 @@ + new \Apps\Core\Packages\Devtools\Modules\Install\Schema\FilesHash, 'model' => new \Apps\Core\Packages\Devtools\Modules\Model\AppsCoreDevtoolsFilesHash, ]; + $schema['devtools_test'] = [ + 'schema' => new \Apps\Core\Packages\Devtools\Test\Install\Schema\DevtoolsTest, + 'model' => new \Apps\Core\Packages\Devtools\Test\Model\DevtoolsTest, + ]; } return $schema;