From 596c05562ecf8a2c3838e58783c96d5be7832f03 Mon Sep 17 00:00:00 2001 From: Frederico Rodrigues Date: Fri, 11 Apr 2025 18:29:29 -0400 Subject: [PATCH 1/7] InstanceExport.ps1: Updating script to enable multi instances export I've added the $instance parameter to a few Cmdlets and updated the logic of the Export code so we get the manifest folder path and look for each one of the instances manifest json files and create a folder for each instance that will be exported inside the exportFilePath people provide when running the script. Also updated the logging so it's friendly to users enough to understand where the script is at at any point in time. --- InstanceExport/PowerShell/InstanceExport.ps1 | 278 ++++++++++--------- 1 file changed, 146 insertions(+), 132 deletions(-) diff --git a/InstanceExport/PowerShell/InstanceExport.ps1 b/InstanceExport/PowerShell/InstanceExport.ps1 index a5ff759..606db42 100644 --- a/InstanceExport/PowerShell/InstanceExport.ps1 +++ b/InstanceExport/PowerShell/InstanceExport.ps1 @@ -68,7 +68,8 @@ Function Log { [Parameter(Mandatory = $true)][String]$msg, [Parameter(Mandatory = $false)][String]$displayMsg, [Parameter(Mandatory = $true)][String]$logLevel, - [Parameter(Mandatory = $true)][String]$currentDate + [Parameter(Mandatory = $true)][String]$currentDate, + [Parameter(Mandatory = $true)][String]$instance ) switch ($logLevel) { "error" { @@ -92,25 +93,26 @@ Function Log { } } } - Add-Content "$($exportFilePath)/InstanceExport_$($currentDate)_log.txt" $msg + Add-Content "$($exportFilePath)/$instance/$($instance)_InstanceExport_$($currentDate)_log.txt" $msg } Function CreateDirectoryIfDoesNotExist { param ( [Parameter(Mandatory = $true)][String]$directoryPath, - [Parameter(Mandatory = $true)][String] $currentDate + [Parameter(Mandatory = $true)][String] $currentDate, + [Parameter(Mandatory = $true)][String] $instance ) $directoryExists = Test-Path -Path $directoryPath if (!$directoryExists) { try { New-Item -Path $directoryPath -ItemType "directory" | Out-Null $msg = "$directoryPath created" - Log -msg $msg -displayMsg $msg -logLevel "info" -currentDate $currentDate + Log -msg $msg -displayMsg $msg -logLevel "info" -currentDate $currentDate -instance $instance } catch { $msg = "Error trying to create $directoryPath directory" $displayMsg = "Error creating directory" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate -instance $instance } } } @@ -121,7 +123,7 @@ Function GetPassword() { return ConvertFrom-SecureString -SecureString $password -AsPlainText } catch { - Log -msg $_.Exception.Message -logLevel "displayInfo" -currentDate $currentDate + Log -msg $_.Exception.Message -logLevel "displayInfo" -currentDate $currentDate -instance $instance } # fallback to powershell 5 $BSTR = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($password) @@ -131,7 +133,7 @@ Function GetPassword() { } # we're probably on some non-windows environment... fall back to unsecured $msg = "Warning: Unable to handle password securely. Falling back to plain text password" - Log -msg $msg -logLevel "displayInfo" -currentDate $currentDate + Log -msg $msg -logLevel "displayInfo" -currentDate $currentDate -instance $instance return Read-Host "$msg. To continue, please enter the password again" } @@ -187,150 +189,162 @@ function Export { This function exports an given instancce and save it to your disk in the specified exportFilePath from a JSON manifest file #> $currentDate = Get-Date -Format "MM_dd_yyyy_HH_mm_ss" - - $manifestFileExists = Test-Path -Path $manifestFilePath - - if ($manifestFileExists) { - $manifestJson = Get-Content -Raw -Path $manifestFilePath | ConvertFrom-Json - if ($manifestJson.CreateDate) { - $currentDate = Get-Date $manifestJson.CreateDate -Format "MM_dd_yyyy_HH_mm_ss" - } - else { - $msg = "Error in reading manifest file" - $displayMsg = "An error occurred when reading manifest file" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate - Exit 1 - } - - try { - CreateDirectoryIfDoesNotExist -directoryPath $exportFilePath -currentDate $currentDate - } - catch { - $msg = "Error trying to create $directoryPath directory" - $displayMsg = "An error occurred when trying to create a new directory" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate - Exit 1 - } - - try { - $accessTokenResponseModel = Login - } - catch { - Log -msg "An error occurred" -displayMsg "$($_.Exception.Response.StatusCode.value__): An error occurred when logging in" -logLevel "error" -currentDate $currentDate - Log -msg "Authorization error for Instance Export" -logLevel "error" -currentDate $currentDate - Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate - Log -msg "Url: api/login" -logLevel "error" -currentDate $currentDate - Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate - } - - if ($accessTokenResponseModel) { - - $accessToken = $accessTokenResponseModel.access_token - - $header = @{ - "authorization" = "Bearer $accessToken" - } - - $currentCategory = ""; - $i = 0; - for (; $i -lt $manifestJson.Entries.Length; $i = $i + 1) { - $entry = $manifestJson.Entries[$i] - if ($currentCategory -ne $entry.Category) { - $currentCategory = $entry.Category; - $message = "Starting extraction of $($entry.Category) category." - Log -msg $message -displayMsg $message -logLevel "displayInfo" -currentDate $currentDate + + $manifestFilePathExists = Test-Path -Path $manifestFilePath + + if ($manifestFilePathExists) { + $manifestFilePath = $manifestFilePath.TrimEnd('\') # Trim any trailing backslash if it exists + + $manifesJsonFiles = Get-ChildItem -Path "$manifestFilePath\*" -Include "*.json" + + foreach ($manifestFile in $manifesJsonFiles) { + + $manifestFileExists = Test-Path -Path $manifestFile + + $instance = ($manifestFile.BaseName -split ' ')[0] + + if ($manifestFileExists) { + $manifestJson = Get-Content -Raw -Path $manifestFile | ConvertFrom-Json + if ($manifestJson.CreateDate) { + $currentDate = Get-Date $manifestJson.CreateDate -Format "MM_dd_yyyy_HH_mm_ss" + } + else { + $msg = "Error in reading manifest file" + $displayMsg = "An error occurred when reading manifest file" + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate -instance $instance + Exit 1 } - $message = "Extracting data from $($entry.Url)" - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - $directoryPath = $exportFilePath + "/" + $entry.Path - + try { - CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate + CreateDirectoryIfDoesNotExist -directoryPath "$exportFilePath\$instance" -currentDate $currentDate -instance $instance } catch { $msg = "Error trying to create $directoryPath directory" $displayMsg = "An error occurred when trying to create a new directory" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate -instance $instance + Exit 1 + } + + $msg = "$instance" + $displayMsg = "Starting exporting data for instance $instance" + Log -msg $msg -displayMsg $displayMsg -logLevel "success" -currentDate $currentDate -instance $instance + + try { + $accessTokenResponseModel = Login + } + catch { + Log -msg "An error occurred" -displayMsg "$($_.Exception.Response.StatusCode.value__): An error occurred when logging in" -logLevel "error" -currentDate $currentDate -instance $instance + Log -msg "Authorization error for Instance Export" -logLevel "error" -currentDate $currentDate -instance $instance + Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate -instance $instance + Log -msg "Url: api/login" -logLevel "error" -currentDate $currentDate -instance $instance + Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate -instance $instance } - $fileName = $entry.FileName.Split([IO.Path]::GetInvalidFileNameChars()) -join '' - $filePath = "$directoryPath/$fileName" - $fileAlreadyExists = Test-Path -Path $filePath -PathType Leaf - if (!$fileAlreadyExists -or $overwrite) { - $entryExportParameters = @{ - Method = "GET" - Uri = "https://$($manifestJson.SubDomain).$($manifestJson.HostName)$($entry.Url)" - Headers = $header - ContentType = "application/json" + + if ($accessTokenResponseModel) { + + $accessToken = $accessTokenResponseModel.access_token + + $header = @{ + "authorization" = "Bearer $accessToken" } - - try { - if (!$entry.fileName.Contains(".json")) { - Invoke-RestMethod @entryExportParameters -OutFile $filePath | Out-Null + + $currentCategory = ""; + $i = 0; + for (; $i -lt $manifestJson.Entries.Length; $i = $i + 1) { + $entry = $manifestJson.Entries[$i] + if ($currentCategory -ne $entry.Category) { + $currentCategory = $entry.Category; + $message = "Starting extraction of $($entry.Category) category." + Log -msg $message -displayMsg $message -logLevel "displayInfo" -currentDate $currentDate -instance $instance + } + $message = "Extracting data from $($entry.Url)" + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate -instance $instance + $directoryPath = $exportFilePath + "/" + $instance + "/" + $entry.Path + + try { + CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate -instance $instance + } + catch { + $msg = "Error trying to create $directoryPath directory" + $displayMsg = "An error occurred when trying to create a new directory" + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate -instance $instance + } + $fileName = $entry.FileName.Split([IO.Path]::GetInvalidFileNameChars()) -join '' + $filePath = "$directoryPath/$fileName" + $fileAlreadyExists = Test-Path -Path $filePath -PathType Leaf + if (!$fileAlreadyExists -or $overwrite) { + $entryExportParameters = @{ + Method = "GET" + Uri = "https://$($manifestJson.SubDomain).$($manifestJson.HostName)$($entry.Url)" + Headers = $header + ContentType = "application/json" + } + + try { + if (!$entry.fileName.Contains(".json")) { + Invoke-RestMethod @entryExportParameters -OutFile $filePath | Out-Null + } + else { + $GetEntriesResponse = Invoke-RestMethod @entryExportParameters + $GetEntriesResponse | ConvertTo-Json -Depth 100 | Out-File -FilePath $filePath + } + } + catch { + $message = "Error occurred when extracting data from $($entry.Url)" + Log -msg "An error occurred" -displayMsg $message -logLevel "error" -currentDate $currentDate -instance $instance + Log -msg $message -logLevel "error" -currentDate $currentDate -instance $instance + Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate -instance $instance + Log -msg "Url: $($entry.Url)" -logLevel "error" -currentDate $currentDate -instance $instance + Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate -instance $instance + Log -msg "Exception: $($_.Exception)" -logLevel "error" -currentDate $currentDate -instance $instance + Write-Verbose $_.Exception + Write-Verbose $_.Exception.Response + if ($_.Exception.Response.StatusCode -eq 503) { + $message = "Lost connection to server. Waiting to restablish connection." + Write-ColorOutput red $message + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + do { + $response = ServerHealtCheck("https://$($manifestJson.SubDomain).$($manifestJson.HostName)/en/healthcheck"); + } while ($response.Exception) + $message = "Server connection reestablished. Resuming export" + Write-ColorOutput green $message + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + $i = $i - 1; + } + } } else { - $GetEntriesResponse = Invoke-RestMethod @entryExportParameters - $GetEntriesResponse | ConvertTo-Json -Depth 100 | Out-File -FilePath $filePath + $message = "Skipping $filePath because file already exists" + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate -instance $instance } } + $msg = "Exporting data for instance $instance was succesful." + Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate -instance $instance + + try { + Logout($header) + } catch { - $message = "Error occurred when extracting data from $($entry.Url)" - Log -msg "An error occurred" -displayMsg $message -logLevel "error" -currentDate $currentDate - Log -msg $message -logLevel "error" -currentDate $currentDate - Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate - Log -msg "Url: $($entry.Url)" -logLevel "error" -currentDate $currentDate - Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate - Log -msg "Exception: $($_.Exception)" -logLevel "error" -currentDate $currentDate - Write-Verbose $_.Exception - Write-Verbose $_.Exception.Response - if ($_.Exception.Response.StatusCode -eq 503) { - $message = "Lost connection to server. Waiting to restablish connection." - Write-ColorOutput red $message - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - do { - $response = ServerHealtCheck("https://$($manifestJson.SubDomain).$($manifestJson.HostName)/en/healthcheck"); - - } while ($response.Exception) - $message = "Server connection reestablished. Resuming export" - Write-ColorOutput green $message - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - $i = $i - 1; - } + $message = "Error occurred when logging out" + Log -msg $message -displayMsg $message -logLevel "error" -currentDate $currentDate -instance $instance } } else { - $message = "Skipping $filePath because file already exists" - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + $msg = "Exporting data for instance $instance finished with errors." + $displayMsg = "Exporting data for instance $instance was not successful. Check the log file for more details." + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate -instance $instance } + } - $msg = "Exporting Instance run finished." - Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate - - try { - Logout($header) - } - catch { - $message = "Error occurred when logging out" - Log -msg $message -displayMsg $message -logLevel "error" -currentDate $currentDate + else { + $displayMsg = "An error occurred when looking for manifest folder" + Log -msg "Error in finding manifest folder" -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate } + } - else { - $msg = "Exporting Instance run finished with errors." - $displayMsg = "Exporting Instance run was not successful. Check the log file for more details." - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate - } - - } - else { - try { - CreateDirectoryIfDoesNotExist -directoryPath $exportFilePath -currentDate $currentDate - } - catch { - $displayMsg = "An error occurred when trying to create a new directory" - Log -msg "Error trying to create $directoryPath directory" -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate - } - - $displayMsg = "An error occurred when looking for manifest file" - Log -msg "Error in finding manifest file" -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + $msg = "Exporting Instance script run finished." + Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate -instance $instance } + } -Export +Export \ No newline at end of file From 8c272446a938adb96fdc9ffa7a256e61f3ae2650 Mon Sep 17 00:00:00 2001 From: Frederico Rodrigues Date: Fri, 11 Apr 2025 19:28:16 -0400 Subject: [PATCH 2/7] README.md: Updating texts for instructions on how to run the script The images were not updated yet. I'll do it using the github web version. More commits should be coming to update the readme images --- InstanceExport/PowerShell/README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/InstanceExport/PowerShell/README.md b/InstanceExport/PowerShell/README.md index 58ada57..c550cda 100644 --- a/InstanceExport/PowerShell/README.md +++ b/InstanceExport/PowerShell/README.md @@ -4,13 +4,17 @@ At DevResults we value the concept that your data belongs to _you_, and you have In order to use it, you should: -1. Download the [InstanceExport.ps1](https://github.com/DevResults/DevResultsTools/releases/download/1.0.2/InstanceExport.ps1) PowerShell script. +1. Download the [InstanceExport.ps1](https://github.com/DevResults/DevResultsTools/releases/download/1.0.3/InstanceExport.ps1) PowerShell script. -2. Reach out to us at help@devresults.com to request an Instance Export Manifest. +2. Reach out to us at help@devresults.com to request an Instance Export Manifest. Or Manifests in case you need to export more than one instance you own. -3. Save the manifest file in the same directory/folder you have saved the powershell script. It's important to save the file in JSON format and to remember the name of the file. In this tutorial the name we use is _manifest.json_. +3. Save the `InstanceExport.ps1` file in a directory in your machine like `C:\Users\MyUser\InstanceExport`. -4. Open a new command line interface (CLI) prompt that supports PowerShell commands (depending on your organizational IT policies, you may need to open an elevated/administrator prompt, e.g. by right clicking on the Powershell icon and choosing "Run as administrator"). If you don't have PowerShell installed you can follow instructions at [Installing Power Shell](https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.3). +4. Inside the folder created on step 3. create a `manifest` folder. + +5. Save your manifest file(s) inside the `manifest` folder. The manifest files come in the JSON format and they are necessary to be that way to be read by the script. In this tutorial consider that your file(s) will be named in the pattern yourInstanceName_manifest.json_. + +4. Open a new command line interface (CLI) prompt that supports PowerShell commands (depending on your organizational IT policies, you may need to open an elevated/administrator prompt, e.g. by right clicking on the Powershell icon and choosing "Run as administrator"). If you don't have PowerShell installed you can follow instructions at [Installing Power Shell](https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.5). Please note that there will be subtle differences between `Windows PowerShell` and `PowerShell 7` command prompts: @@ -22,7 +26,7 @@ Please note that there will be subtle differences between `Windows PowerShell` a ![image](https://user-images.githubusercontent.com/67288628/225463265-13a63f36-ef92-4813-9108-e4e949dc8e3f.png) -5. Navigate to the directory where you have saved the InstanceExport.ps1 and the JSON manifest file. +6. Navigate to the directory where you have saved the `InstanceExport.ps1` and that has also the `manifest` folder. e.g.: `cd C:\Users\MyUser\InstanceExport` @@ -38,8 +42,8 @@ It will run automatically after you enter all required fields. The progress of y The PowerShell script has five parameters that are explained below: -- manifestFilePath: Path (or just file name, if same directory) of the manifest file you have downloaded using step 2, e.g. `C:\Users\MyUser\InstanceExport\manifest.json`. -- exportFilePath: Path (or just file name, if same directory) to create a folder for the exported files, e.g. `C:\Users\MyUser\InstanceExport\2021_Export\`; you do not need to create this folder manually, the script will do so for you. +- manifestFilePath: Path (or just folder name, if in the same directory) of the manifest file(s) you have added inside the `manifest` folder from step 3, e.g. `C:\Users\MyUser\InstanceExport\manifest`. +- exportFilePath: Path (or just folder name, if in the same directory) to create a folder that will contain all instance(s) you are going to export, e.g. `C:\Users\MyUser\InstanceExport\2025_Export\`; you do not need to create this folder manually, the script will do so for you. - userName: Your username (work email) for login at DevResults, e.g. `first.last@org.org`. - password: Your password for login at DevResults or API Key's Secret. (If you're using a password manager and copy/pasting your password, be aware that CTRL-V does not work in Powershell; try right clicking in the window to paste instead.) - overwrite: Optional parameter to confirm whether or not you want to overwrite files that already exist and replace them. This field defaults to `false`, which means that if a file already exists in the exportFilePath, it will be skipped. From fe30b37a560696802450857cdc5bf7b3c9a3e692 Mon Sep 17 00:00:00 2001 From: Frederico Rodrigues Lima de Souza Pinto Date: Fri, 11 Apr 2025 20:00:24 -0400 Subject: [PATCH 3/7] Update README.md Updating images and texts explaining the possibility of exporting multiple instances in a single run of InstanceExport script --- InstanceExport/PowerShell/README.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/InstanceExport/PowerShell/README.md b/InstanceExport/PowerShell/README.md index c550cda..208be6a 100644 --- a/InstanceExport/PowerShell/README.md +++ b/InstanceExport/PowerShell/README.md @@ -20,11 +20,11 @@ Please note that there will be subtle differences between `Windows PowerShell` a `Windows PowerShell` usually comes installed with Windows. It uses the `PowerShell 5` version as you can see from the image below: -![image](https://user-images.githubusercontent.com/67288628/225462134-9a8e0224-3638-46be-9758-5adaf401d655.png) +![image](https://github.com/user-attachments/assets/3f61a5d3-8b95-43cb-b85d-d030b598aa1f) `PowerShell 7` is the most recent version of PowerShell released by Microsoft. We recommend using it as it supports longer file paths, which allows our `InstanceExport` script to export files with long names properly. After you install `PowerShell 7` and open its command line prompt, you should be able to see the version of PowerShell using the `$PSVersionTable.PSVersion` command as shown in the image below: -![image](https://user-images.githubusercontent.com/67288628/225463265-13a63f36-ef92-4813-9108-e4e949dc8e3f.png) +![image](https://github.com/user-attachments/assets/9e7449fb-fe9d-414c-8b93-eaaf7f4ded0b) 6. Navigate to the directory where you have saved the `InstanceExport.ps1` and that has also the `manifest` folder. @@ -36,7 +36,7 @@ Please note that there will be subtle differences between `Windows PowerShell` a The process will prompt you to enter the required fields: `$manifestFilePath`, `$exportFilePath`, `$userName` and `$password`. Because you navigated to the directory/folder in step 5, you do not need to enter the full path, just the file names (see example below). -![image](https://user-images.githubusercontent.com/67288628/225464180-819117b1-0f24-4ecb-a6c7-ae2d45db34d6.png) +![image](https://github.com/user-attachments/assets/eca84717-d4b1-40a6-a8ef-415f11bdc546) It will run automatically after you enter all required fields. The progress of your data export will be logged by each available category. When all is finished you should see the message "Exporting Instance finished". @@ -50,12 +50,25 @@ The PowerShell script has five parameters that are explained below: Please note that if you use `.\InstanceExport.ps1` without passing any parameters you will be asked to provide the information and will not be able to use the overwrite parameter. If you are more experienced with PowerShell, you can also use the command by passing parameters as shown in the image below: -![image](https://user-images.githubusercontent.com/67288628/225468832-d4fc83d7-4980-45b4-8a69-094f17e67b0d.png) +![image](https://github.com/user-attachments/assets/de040cf5-f268-4316-8ef5-2276b1e72648) ### Output We expect that things will go smoothly while you are using the DevResults InstanceExport script and that you get a result similar to the following image: -![image](https://user-images.githubusercontent.com/67288628/225465649-ac48360f-af6c-458b-a294-c0e0409d33e3.png) +![image](https://github.com/user-attachments/assets/0af5a12c-4c5e-4fc2-b737-1901d2d5739f) + +In the folder you've saved the `InstanceExport` script you will noticed that a new folder `export` (or other name you've provided in the $exportFilePath parameter) and inside that folder you will have a folder with the name of the exported instance. You should expect that your data exported will be there in subfolders and a log file will be generated with the pattern `yourInstance_InstanceExport_MM_dd_yyyy_HH_mm_ss_log.txt` + +![image](https://github.com/user-attachments/assets/2078418b-77ea-4131-9f4a-3f075cc3b300) + +In case you are exporting more than one instance you will have a similar output but with more log information like the following image: + +![image](https://github.com/user-attachments/assets/9f425cd6-34b2-449a-af71-bdc2de9bae6d) + +And similarly you will have as many folders as manifests files you have inside your `manifest` folder. You will notice that the script would be generating one folder for each instance you've own. + +![image](https://github.com/user-attachments/assets/be64f831-f668-45e0-aceb-7bb1d153e0b4) + ### Troubleshooting From 3411cc134408d0afcd225735922d50d945b3ffd0 Mon Sep 17 00:00:00 2001 From: Frederico Rodrigues Date: Tue, 15 Apr 2025 14:05:26 -0400 Subject: [PATCH 4/7] InstanceExport: Removing $instance on Log Cmdlet Also adding code to check if the manifestFilePath is a json file or it's a folder. If it's a json file it runs smoothly as before and if it's a folder try to find as many as manifests that are available there. Also updating $instance value to be coming from the subdomain and not relying in the manifest file name. --- InstanceExport/PowerShell/InstanceExport.ps1 | 403 +++++++++++++------ 1 file changed, 275 insertions(+), 128 deletions(-) diff --git a/InstanceExport/PowerShell/InstanceExport.ps1 b/InstanceExport/PowerShell/InstanceExport.ps1 index 606db42..2218084 100644 --- a/InstanceExport/PowerShell/InstanceExport.ps1 +++ b/InstanceExport/PowerShell/InstanceExport.ps1 @@ -68,8 +68,7 @@ Function Log { [Parameter(Mandatory = $true)][String]$msg, [Parameter(Mandatory = $false)][String]$displayMsg, [Parameter(Mandatory = $true)][String]$logLevel, - [Parameter(Mandatory = $true)][String]$currentDate, - [Parameter(Mandatory = $true)][String]$instance + [Parameter(Mandatory = $true)][String]$currentDate ) switch ($logLevel) { "error" { @@ -93,26 +92,25 @@ Function Log { } } } - Add-Content "$($exportFilePath)/$instance/$($instance)_InstanceExport_$($currentDate)_log.txt" $msg + Add-Content "$($exportFilePath)/InstanceExport_$($currentDate)_log.txt" $msg } Function CreateDirectoryIfDoesNotExist { param ( [Parameter(Mandatory = $true)][String]$directoryPath, - [Parameter(Mandatory = $true)][String] $currentDate, - [Parameter(Mandatory = $true)][String] $instance + [Parameter(Mandatory = $true)][String] $currentDate ) $directoryExists = Test-Path -Path $directoryPath if (!$directoryExists) { try { New-Item -Path $directoryPath -ItemType "directory" | Out-Null $msg = "$directoryPath created" - Log -msg $msg -displayMsg $msg -logLevel "info" -currentDate $currentDate -instance $instance + Log -msg $msg -displayMsg $msg -logLevel "info" -currentDate $currentDate } catch { $msg = "Error trying to create $directoryPath directory" $displayMsg = "Error creating directory" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate -instance $instance + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate } } } @@ -123,7 +121,7 @@ Function GetPassword() { return ConvertFrom-SecureString -SecureString $password -AsPlainText } catch { - Log -msg $_.Exception.Message -logLevel "displayInfo" -currentDate $currentDate -instance $instance + Log -msg $_.Exception.Message -logLevel "displayInfo" -currentDate $currentDate } # fallback to powershell 5 $BSTR = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($password) @@ -133,7 +131,7 @@ Function GetPassword() { } # we're probably on some non-windows environment... fall back to unsecured $msg = "Warning: Unable to handle password securely. Falling back to plain text password" - Log -msg $msg -logLevel "displayInfo" -currentDate $currentDate -instance $instance + Log -msg $msg -logLevel "displayInfo" -currentDate $currentDate return Read-Host "$msg. To continue, please enter the password again" } @@ -192,159 +190,308 @@ This function exports an given instancce and save it to your disk in the specifi $manifestFilePathExists = Test-Path -Path $manifestFilePath + $target = Get-Item $manifestFilePath + if ($manifestFilePathExists) { - $manifestFilePath = $manifestFilePath.TrimEnd('\') # Trim any trailing backslash if it exists - - $manifesJsonFiles = Get-ChildItem -Path "$manifestFilePath\*" -Include "*.json" - - foreach ($manifestFile in $manifesJsonFiles) { + if (!$target.PSIsContainer) { + $manifestJson = Get-Content -Raw -Path $manifestFilePath | ConvertFrom-Json + $instance = $manifestJson.subdomain - $manifestFileExists = Test-Path -Path $manifestFile - - $instance = ($manifestFile.BaseName -split ' ')[0] - - if ($manifestFileExists) { - $manifestJson = Get-Content -Raw -Path $manifestFile | ConvertFrom-Json - if ($manifestJson.CreateDate) { - $currentDate = Get-Date $manifestJson.CreateDate -Format "MM_dd_yyyy_HH_mm_ss" - } - else { - $msg = "Error in reading manifest file" - $displayMsg = "An error occurred when reading manifest file" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate -instance $instance - Exit 1 - } - - try { - CreateDirectoryIfDoesNotExist -directoryPath "$exportFilePath\$instance" -currentDate $currentDate -instance $instance - } - catch { - $msg = "Error trying to create $directoryPath directory" - $displayMsg = "An error occurred when trying to create a new directory" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate -instance $instance - Exit 1 + if ($manifestJson.CreateDate) { + $currentDate = Get-Date $manifestJson.CreateDate -Format "MM_dd_yyyy_HH_mm_ss" + } + else { + $msg = "Error in reading manifest file" + $displayMsg = "An error occurred when reading manifest file" + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + Exit 1 + } + + $directoryPath = $exportFilePath + "/" + $instance + try { + CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate + } + catch { + $msg = "Error trying to create $directoryPath directory" + $displayMsg = "An error occurred when trying to create a new directory" + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + Exit 1 + } + + try { + $accessTokenResponseModel = Login + } + catch { + Log -msg "An error occurred" -displayMsg "$($_.Exception.Response.StatusCode.value__): An error occurred when logging in" -logLevel "error" -currentDate $currentDate + Log -msg "Authorization error for Instance Export" -logLevel "error" -currentDate $currentDate + Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate + Log -msg "Url: api/login" -logLevel "error" -currentDate $currentDate + Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate + } + + if ($accessTokenResponseModel) { + + $accessToken = $accessTokenResponseModel.access_token + + $header = @{ + "authorization" = "Bearer $accessToken" } $msg = "$instance" $displayMsg = "Starting exporting data for instance $instance" - Log -msg $msg -displayMsg $displayMsg -logLevel "success" -currentDate $currentDate -instance $instance + Log -msg $msg -displayMsg $displayMsg -logLevel "success" -currentDate $currentDate + + $currentCategory = ""; + $i = 0; + for (; $i -lt $manifestJson.Entries.Length; $i = $i + 1) { + $entry = $manifestJson.Entries[$i] + if ($currentCategory -ne $entry.Category) { + $currentCategory = $entry.Category; + $message = "Starting extraction of $($entry.Category) category." + Log -msg $message -displayMsg $message -logLevel "displayInfo" -currentDate $currentDate + } + $message = "Extracting data from $($entry.Url)" + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + $directoryPath = $exportFilePath + "/" + $instance + "/" + $entry.Path + + try { + CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate + } + catch { + $msg = "Error trying to create $directoryPath directory" + $displayMsg = "An error occurred when trying to create a new directory" + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + } + $fileName = $entry.FileName.Split([IO.Path]::GetInvalidFileNameChars()) -join '' + $filePath = "$directoryPath/$fileName" + $fileAlreadyExists = Test-Path -Path $filePath -PathType Leaf + if (!$fileAlreadyExists -or $overwrite) { + $entryExportParameters = @{ + Method = "GET" + Uri = "https://$($manifestJson.SubDomain).$($manifestJson.HostName)$($entry.Url)" + Headers = $header + ContentType = "application/json" + } + + try { + if (!$entry.fileName.Contains(".json")) { + Invoke-RestMethod @entryExportParameters -OutFile $filePath | Out-Null + } + else { + $GetEntriesResponse = Invoke-RestMethod @entryExportParameters + $GetEntriesResponse | ConvertTo-Json -Depth 100 | Out-File -FilePath $filePath + } + } + catch { + $message = "Error occurred when extracting data from $($entry.Url)" + Log -msg "An error occurred" -displayMsg $message -logLevel "error" -currentDate $currentDate + Log -msg $message -logLevel "error" -currentDate $currentDate + Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate + Log -msg "Url: $($entry.Url)" -logLevel "error" -currentDate $currentDate + Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate + Log -msg "Exception: $($_.Exception)" -logLevel "error" -currentDate $currentDate + Write-Verbose $_.Exception + Write-Verbose $_.Exception.Response + if ($_.Exception.Response.StatusCode -eq 503) { + $message = "Lost connection to server. Waiting to restablish connection." + Write-ColorOutput red $message + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + do { + $response = ServerHealtCheck("https://$($manifestJson.SubDomain).$($manifestJson.HostName)/en/healthcheck"); + + } while ($response.Exception) + $message = "Server connection reestablished. Resuming export" + Write-ColorOutput green $message + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + $i = $i - 1; + } + } + } + else { + $message = "Skipping $filePath because file already exists" + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + } + } + $msg = "Exporting Instance run finished." + Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate try { - $accessTokenResponseModel = Login + Logout($header) } catch { - Log -msg "An error occurred" -displayMsg "$($_.Exception.Response.StatusCode.value__): An error occurred when logging in" -logLevel "error" -currentDate $currentDate -instance $instance - Log -msg "Authorization error for Instance Export" -logLevel "error" -currentDate $currentDate -instance $instance - Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate -instance $instance - Log -msg "Url: api/login" -logLevel "error" -currentDate $currentDate -instance $instance - Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate -instance $instance + $message = "Error occurred when logging out" + Log -msg $message -displayMsg $message -logLevel "error" -currentDate $currentDate } - - if ($accessTokenResponseModel) { - - $accessToken = $accessTokenResponseModel.access_token - - $header = @{ - "authorization" = "Bearer $accessToken" - } - - $currentCategory = ""; - $i = 0; - for (; $i -lt $manifestJson.Entries.Length; $i = $i + 1) { - $entry = $manifestJson.Entries[$i] - if ($currentCategory -ne $entry.Category) { - $currentCategory = $entry.Category; - $message = "Starting extraction of $($entry.Category) category." - Log -msg $message -displayMsg $message -logLevel "displayInfo" -currentDate $currentDate -instance $instance - } - $message = "Extracting data from $($entry.Url)" - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate -instance $instance - $directoryPath = $exportFilePath + "/" + $instance + "/" + $entry.Path + } + else { + $msg = "Exporting Instance run finished with errors." + $displayMsg = "Exporting Instance run was not successful. Check the log file for more details." + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + } + } + else { + $manifestFilePath = $manifestFilePath.TrimEnd('\') # Trim any trailing backslash if it exists + + $manifestJsonFiles = Get-ChildItem -Path "$manifestFilePath\*" -Include "*.json" + + try { + CreateDirectoryIfDoesNotExist -directoryPath "$exportFilePath" -currentDate $currentDate + } + catch { + $msg = "Error trying to create $directoryPath directory" + $displayMsg = "An error occurred when trying to create a new directory" + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + Exit 1 + } + + if ($manifestJsonFiles) { + + foreach ($manifestFile in $manifestJsonFiles) { + $manifestFileExists = Test-Path -Path $manifestFile + + if ($manifestFileExists) { + $manifestJson = Get-Content -Raw -Path $manifestFile | ConvertFrom-Json + + $instance = $manifestJson.subdomain + try { - CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate -instance $instance + CreateDirectoryIfDoesNotExist -directoryPath "$exportFilePath\$instance" -currentDate $currentDate } catch { $msg = "Error trying to create $directoryPath directory" $displayMsg = "An error occurred when trying to create a new directory" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate -instance $instance + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + Exit 1 } - $fileName = $entry.FileName.Split([IO.Path]::GetInvalidFileNameChars()) -join '' - $filePath = "$directoryPath/$fileName" - $fileAlreadyExists = Test-Path -Path $filePath -PathType Leaf - if (!$fileAlreadyExists -or $overwrite) { - $entryExportParameters = @{ - Method = "GET" - Uri = "https://$($manifestJson.SubDomain).$($manifestJson.HostName)$($entry.Url)" - Headers = $header - ContentType = "application/json" + + try { + $accessTokenResponseModel = Login + } + catch { + Log -msg "An error occurred" -displayMsg "$($_.Exception.Response.StatusCode.value__): An error occurred when logging in" -logLevel "error" -currentDate $currentDate + Log -msg "Authorization error for Instance Export" -logLevel "error" -currentDate $currentDate + Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate + Log -msg "Url: api/login" -logLevel "error" -currentDate $currentDate + Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate + } + + if ($accessTokenResponseModel) { + + $accessToken = $accessTokenResponseModel.access_token + + $header = @{ + "authorization" = "Bearer $accessToken" } + + $message = "Starting exporting data for instance $instance" + Log -msg $message -displayMsg $message -logLevel "success" -currentDate $currentDate + + $currentCategory = ""; + $i = 0; + for (; $i -lt $manifestJson.Entries.Length; $i = $i + 1) { + $entry = $manifestJson.Entries[$i] + if ($currentCategory -ne $entry.Category) { + $currentCategory = $entry.Category; + $message = "Starting extraction of $($entry.Category) category." + Log -msg $message -displayMsg $message -logLevel "displayInfo" -currentDate $currentDate + } + $message = "Extracting data from $($entry.Url)" + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + $directoryPath = $exportFilePath + "/" + $instance + "/" + $entry.Path - try { - if (!$entry.fileName.Contains(".json")) { - Invoke-RestMethod @entryExportParameters -OutFile $filePath | Out-Null + try { + CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate + } + catch { + $msg = "Error trying to create $directoryPath directory" + $displayMsg = "An error occurred when trying to create a new directory" + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + } + $fileName = $entry.FileName.Split([IO.Path]::GetInvalidFileNameChars()) -join '' + $filePath = "$directoryPath/$fileName" + $fileAlreadyExists = Test-Path -Path $filePath -PathType Leaf + if (!$fileAlreadyExists -or $overwrite) { + $entryExportParameters = @{ + Method = "GET" + Uri = "https://$($manifestJson.SubDomain).$($manifestJson.HostName)$($entry.Url)" + Headers = $header + ContentType = "application/json" + } + + try { + if (!$entry.fileName.Contains(".json")) { + Invoke-RestMethod @entryExportParameters -OutFile $filePath | Out-Null + } + else { + $GetEntriesResponse = Invoke-RestMethod @entryExportParameters + $GetEntriesResponse | ConvertTo-Json -Depth 100 | Out-File -FilePath $filePath + } + } + catch { + $message = "Error occurred when extracting data from $($entry.Url)" + Log -msg "An error occurred" -displayMsg $message -logLevel "error" -currentDate $currentDate + Log -msg $message -logLevel "error" -currentDate $currentDate + Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate + Log -msg "Url: $($entry.Url)" -logLevel "error" -currentDate $currentDate + Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate + Log -msg "Exception: $($_.Exception)" -logLevel "error" -currentDate $currentDate + Write-Verbose $_.Exception + Write-Verbose $_.Exception.Response + if ($_.Exception.Response.StatusCode -eq 503) { + $message = "Lost connection to server. Waiting to restablish connection." + Write-ColorOutput red $message + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + do { + $response = ServerHealtCheck("https://$($manifestJson.SubDomain).$($manifestJson.HostName)/en/healthcheck"); + } while ($response.Exception) + $message = "Server connection reestablished. Resuming export" + Write-ColorOutput green $message + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + $i = $i - 1; + } + } } else { - $GetEntriesResponse = Invoke-RestMethod @entryExportParameters - $GetEntriesResponse | ConvertTo-Json -Depth 100 | Out-File -FilePath $filePath + $message = "Skipping $filePath because file already exists" + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate } } + $msg = "Exporting data for instance $instance was succesful." + Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate + + try { + Logout($header) + } catch { - $message = "Error occurred when extracting data from $($entry.Url)" - Log -msg "An error occurred" -displayMsg $message -logLevel "error" -currentDate $currentDate -instance $instance - Log -msg $message -logLevel "error" -currentDate $currentDate -instance $instance - Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate -instance $instance - Log -msg "Url: $($entry.Url)" -logLevel "error" -currentDate $currentDate -instance $instance - Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate -instance $instance - Log -msg "Exception: $($_.Exception)" -logLevel "error" -currentDate $currentDate -instance $instance - Write-Verbose $_.Exception - Write-Verbose $_.Exception.Response - if ($_.Exception.Response.StatusCode -eq 503) { - $message = "Lost connection to server. Waiting to restablish connection." - Write-ColorOutput red $message - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - do { - $response = ServerHealtCheck("https://$($manifestJson.SubDomain).$($manifestJson.HostName)/en/healthcheck"); - } while ($response.Exception) - $message = "Server connection reestablished. Resuming export" - Write-ColorOutput green $message - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - $i = $i - 1; - } + $message = "Error occurred when logging out" + Log -msg $message -displayMsg $message -logLevel "error" -currentDate $currentDate } } else { - $message = "Skipping $filePath because file already exists" - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate -instance $instance + $msg = "Exporting data for instance $instance finished with errors." + $displayMsg = "Exporting data for instance $instance was not successful. Check the log file for more details." + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate } } - $msg = "Exporting data for instance $instance was succesful." - Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate -instance $instance - - try { - Logout($header) - } - catch { - $message = "Error occurred when logging out" - Log -msg $message -displayMsg $message -logLevel "error" -currentDate $currentDate -instance $instance + else { + $displayMsg = "An error occurred when looking for manifest file" + Log -msg "Error in finding manifest file" -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate } } - else { - $msg = "Exporting data for instance $instance finished with errors." - $displayMsg = "Exporting data for instance $instance was not successful. Check the log file for more details." - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate -instance $instance - } - - } + } else { - $displayMsg = "An error occurred when looking for manifest folder" - Log -msg "Error in finding manifest folder" -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + $msg = "Manifest file path doesn't contain valid json file(s)." + Log -msg $msg -displayMsg $msg -logLevel "error" -currentDate $currentDate } - + + $msg = "Exporting Instance script run finished." + Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate } - $msg = "Exporting Instance script run finished." - Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate -instance $instance } - + else { + $msg = "Manifest file path provided doesn't exist." + Log -msg $msg -displayMsg $msg -logLevel "error" -currentDate $currentDate + } } -Export \ No newline at end of file +Export From 4bbbe3c5beb7a571bccbfe0e8af7e1c78d6ecc2e Mon Sep 17 00:00:00 2001 From: Frederico Rodrigues Lima de Souza Pinto Date: Tue, 15 Apr 2025 15:27:06 -0400 Subject: [PATCH 5/7] Update README.md Updating README.md file to explain the normal flow for a single instance export and the new flow for multiple instances export flow --- InstanceExport/PowerShell/README.md | 50 ++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/InstanceExport/PowerShell/README.md b/InstanceExport/PowerShell/README.md index 208be6a..f969a6c 100644 --- a/InstanceExport/PowerShell/README.md +++ b/InstanceExport/PowerShell/README.md @@ -10,11 +10,9 @@ In order to use it, you should: 3. Save the `InstanceExport.ps1` file in a directory in your machine like `C:\Users\MyUser\InstanceExport`. -4. Inside the folder created on step 3. create a `manifest` folder. +4. Save the `manifest` file we have provided you inside the same directory of step 3. The manifest file should be always in the `JSON` format in order to script be able to read it. -5. Save your manifest file(s) inside the `manifest` folder. The manifest files come in the JSON format and they are necessary to be that way to be read by the script. In this tutorial consider that your file(s) will be named in the pattern yourInstanceName_manifest.json_. - -4. Open a new command line interface (CLI) prompt that supports PowerShell commands (depending on your organizational IT policies, you may need to open an elevated/administrator prompt, e.g. by right clicking on the Powershell icon and choosing "Run as administrator"). If you don't have PowerShell installed you can follow instructions at [Installing Power Shell](https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.5). +5. Open a new command line interface (CLI) prompt that supports PowerShell commands (depending on your organizational IT policies, you may need to open an elevated/administrator prompt, e.g. by right clicking on the Powershell icon and choosing "Run as administrator"). If you don't have PowerShell installed you can follow instructions at [Installing Power Shell](https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.5). Please note that there will be subtle differences between `Windows PowerShell` and `PowerShell 7` command prompts: @@ -30,12 +28,39 @@ Please note that there will be subtle differences between `Windows PowerShell` a e.g.: `cd C:\Users\MyUser\InstanceExport` -6. In the cli prompt, type the following command. **Be very careful not to click inside the cli prompt while it's running, or it may go into "select mode" and pause.** +7. In the cli prompt, type the following command. **Be very careful not to click inside the cli prompt while it's running, or it may go into "select mode" and pause.** `.\InstanceExport` The process will prompt you to enter the required fields: `$manifestFilePath`, `$exportFilePath`, `$userName` and `$password`. Because you navigated to the directory/folder in step 5, you do not need to enter the full path, just the file names (see example below). +![image](https://github.com/user-attachments/assets/ae36ca77-66f9-4377-84dd-e9fd78417309) + +### Alternative scenario +We also allow to export multiple instances at once using the same `InstanceExport` script. In order to do that possible you should follow almost the same set of instructions: + +1. Download the [InstanceExport.ps1](https://github.com/DevResults/DevResultsTools/releases/download/1.0.3/InstanceExport.ps1) PowerShell script. + +2. Reach out to us at help@devresults.com to request an Instance Export Manifest. Or Manifests in case you need to export more than one instance you own. + +3. Save the `InstanceExport.ps1` file in a directory in your machine like `C:\Users\MyUser\InstanceExport`. + +4. Create a `manifest` folder inside the directory of step 3. + +5. Save all `manifest` files we have provided you inside the directory of step 4. All manifest files should be always in the `JSON` format in order to script be able to read it. + +6. Open a new command line interface (CLI) prompt that supports PowerShell commands (depending on your organizational IT policies, you may need to open an elevated/administrator prompt, e.g. by right clicking on the Powershell icon and choosing "Run as administrator"). + +7. Navigate to the directory where you have saved the `InstanceExport.ps1` and that has also the `manifest` folder. + + e.g.: `cd C:\Users\MyUser\InstanceExport` + +8. In the cli prompt, type the following command. **Be very careful not to click inside the cli prompt while it's running, or it may go into "select mode" and pause.** + + `.\InstanceExport` + +The process will prompt you to enter the required fields: `$manifestFilePath`, `$exportFilePath`, `$userName` and `$password`. Because you navigated to the directory/folder in step 7, you do not need to enter the full path, just the folder names (see example below). + ![image](https://github.com/user-attachments/assets/eca84717-d4b1-40a6-a8ef-415f11bdc546) It will run automatically after you enter all required fields. The progress of your data export will be logged by each available category. When all is finished you should see the message "Exporting Instance finished". @@ -55,20 +80,21 @@ Please note that if you use `.\InstanceExport.ps1` without passing any parameter ### Output We expect that things will go smoothly while you are using the DevResults InstanceExport script and that you get a result similar to the following image: -![image](https://github.com/user-attachments/assets/0af5a12c-4c5e-4fc2-b737-1901d2d5739f) +For single instance Export: -In the folder you've saved the `InstanceExport` script you will noticed that a new folder `export` (or other name you've provided in the $exportFilePath parameter) and inside that folder you will have a folder with the name of the exported instance. You should expect that your data exported will be there in subfolders and a log file will be generated with the pattern `yourInstance_InstanceExport_MM_dd_yyyy_HH_mm_ss_log.txt` +![image](https://github.com/user-attachments/assets/3376daf2-9950-48df-ad54-7f3d1c7ea677) -![image](https://github.com/user-attachments/assets/2078418b-77ea-4131-9f4a-3f075cc3b300) - -In case you are exporting more than one instance you will have a similar output but with more log information like the following image: +For multiple instances Export ![image](https://github.com/user-attachments/assets/9f425cd6-34b2-449a-af71-bdc2de9bae6d) -And similarly you will have as many folders as manifests files you have inside your `manifest` folder. You will notice that the script would be generating one folder for each instance you've own. +In the folder you've saved the `InstanceExport` script you will noticed that a new folder `export` (or other name you've provided in the $exportFilePath parameter) and inside that folder you will have a folder with the name of the exported instance. You should expect that your data exported will be there in subfolders and a log file will be generated with the pattern `InstanceExport_MM_dd_yyyy_HH_mm_ss_log.txt` + +![image](https://github.com/user-attachments/assets/72e839f5-19ce-4cf0-8359-65254245754d) -![image](https://github.com/user-attachments/assets/be64f831-f668-45e0-aceb-7bb1d153e0b4) +And similarly if you followed the process to export more than one instance you will notice in the same folder a folder for each `instance` exported. +![image](https://github.com/user-attachments/assets/e4546e4b-5240-4023-a1fc-b00ad1f33488) ### Troubleshooting From 054acb3207100daef7f2cb65711c967b7018382a Mon Sep 17 00:00:00 2001 From: Frederico Rodrigues Date: Tue, 15 Apr 2025 16:09:59 -0400 Subject: [PATCH 6/7] InstanceExport: Extracting ExportInstanceData So we call it either for the single instance scenario or multiple instances scenario and simplifies the reading a bit --- InstanceExport/PowerShell/InstanceExport.ps1 | 378 +++++++------------ 1 file changed, 135 insertions(+), 243 deletions(-) diff --git a/InstanceExport/PowerShell/InstanceExport.ps1 b/InstanceExport/PowerShell/InstanceExport.ps1 index 2218084..4879565 100644 --- a/InstanceExport/PowerShell/InstanceExport.ps1 +++ b/InstanceExport/PowerShell/InstanceExport.ps1 @@ -180,6 +180,126 @@ Function ServerHealtCheck($Uri) { return $response } +Function ExportInstanceData() { + try { + CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate + } + catch { + $msg = "Error trying to create $directoryPath directory" + $displayMsg = "An error occurred when trying to create a new directory" + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + Exit 1 + } + + try { + $accessTokenResponseModel = Login + } + catch { + Log -msg "An error occurred" -displayMsg "$($_.Exception.Response.StatusCode.value__): An error occurred when logging in" -logLevel "error" -currentDate $currentDate + Log -msg "Authorization error for Instance Export" -logLevel "error" -currentDate $currentDate + Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate + Log -msg "Url: api/login" -logLevel "error" -currentDate $currentDate + Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate + } + if ($accessTokenResponseModel) { + + $accessToken = $accessTokenResponseModel.access_token + + $header = @{ + "authorization" = "Bearer $accessToken" + } + + $message = "Starting exporting data for instance $instance" + Log -msg $message -displayMsg $message -logLevel "success" -currentDate $currentDate + + $currentCategory = ""; + $i = 0; + for (; $i -lt $manifestJson.Entries.Length; $i = $i + 1) { + $entry = $manifestJson.Entries[$i] + if ($currentCategory -ne $entry.Category) { + $currentCategory = $entry.Category; + $message = "Starting extraction of $($entry.Category) category." + Log -msg $message -displayMsg $message -logLevel "displayInfo" -currentDate $currentDate + } + $message = "Extracting data from $($entry.Url)" + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + $directoryPath = $exportFilePath + "/" + $instance + "/" + $entry.Path + + try { + CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate + } + catch { + $msg = "Error trying to create $directoryPath directory" + $displayMsg = "An error occurred when trying to create a new directory" + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + } + $fileName = $entry.FileName.Split([IO.Path]::GetInvalidFileNameChars()) -join '' + $filePath = "$directoryPath/$fileName" + $fileAlreadyExists = Test-Path -Path $filePath -PathType Leaf + if (!$fileAlreadyExists -or $overwrite) { + $entryExportParameters = @{ + Method = "GET" + Uri = "https://$($manifestJson.SubDomain).$($manifestJson.HostName)$($entry.Url)" + Headers = $header + ContentType = "application/json" + } + + try { + if (!$entry.fileName.Contains(".json")) { + Invoke-RestMethod @entryExportParameters -OutFile $filePath | Out-Null + } + else { + $GetEntriesResponse = Invoke-RestMethod @entryExportParameters + $GetEntriesResponse | ConvertTo-Json -Depth 100 | Out-File -FilePath $filePath + } + } + catch { + $message = "Error occurred when extracting data from $($entry.Url)" + Log -msg "An error occurred" -displayMsg $message -logLevel "error" -currentDate $currentDate + Log -msg $message -logLevel "error" -currentDate $currentDate + Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate + Log -msg "Url: $($entry.Url)" -logLevel "error" -currentDate $currentDate + Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate + Log -msg "Exception: $($_.Exception)" -logLevel "error" -currentDate $currentDate + Write-Verbose $_.Exception + Write-Verbose $_.Exception.Response + if ($_.Exception.Response.StatusCode -eq 503) { + $message = "Lost connection to server. Waiting to restablish connection." + Write-ColorOutput red $message + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + do { + $response = ServerHealtCheck("https://$($manifestJson.SubDomain).$($manifestJson.HostName)/en/healthcheck"); + } while ($response.Exception) + $message = "Server connection reestablished. Resuming export" + Write-ColorOutput green $message + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + $i = $i - 1; + } + } + } + else { + $message = "Skipping $filePath because file already exists" + Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate + } + } + try { + Logout($header) + } + catch { + $message = "Error occurred when logging out" + Log -msg $message -displayMsg $message -logLevel "error" -currentDate $currentDate + } + + $msg = "Exporting data for instance $instance was succesful." + Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate + } + else { + $msg = "Exporting Instance run finished with errors." + $displayMsg = "Exporting Instance run was not successful. Check the log file for more details." + Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate + } +} + function Export { <# @@ -208,126 +328,10 @@ This function exports an given instancce and save it to your disk in the specifi } $directoryPath = $exportFilePath + "/" + $instance - try { - CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate - } - catch { - $msg = "Error trying to create $directoryPath directory" - $displayMsg = "An error occurred when trying to create a new directory" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate - Exit 1 - } - - try { - $accessTokenResponseModel = Login - } - catch { - Log -msg "An error occurred" -displayMsg "$($_.Exception.Response.StatusCode.value__): An error occurred when logging in" -logLevel "error" -currentDate $currentDate - Log -msg "Authorization error for Instance Export" -logLevel "error" -currentDate $currentDate - Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate - Log -msg "Url: api/login" -logLevel "error" -currentDate $currentDate - Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate - } - - if ($accessTokenResponseModel) { - - $accessToken = $accessTokenResponseModel.access_token - - $header = @{ - "authorization" = "Bearer $accessToken" - } - - $msg = "$instance" - $displayMsg = "Starting exporting data for instance $instance" - Log -msg $msg -displayMsg $displayMsg -logLevel "success" -currentDate $currentDate - - $currentCategory = ""; - $i = 0; - for (; $i -lt $manifestJson.Entries.Length; $i = $i + 1) { - $entry = $manifestJson.Entries[$i] - if ($currentCategory -ne $entry.Category) { - $currentCategory = $entry.Category; - $message = "Starting extraction of $($entry.Category) category." - Log -msg $message -displayMsg $message -logLevel "displayInfo" -currentDate $currentDate - } - $message = "Extracting data from $($entry.Url)" - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - $directoryPath = $exportFilePath + "/" + $instance + "/" + $entry.Path - - try { - CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate - } - catch { - $msg = "Error trying to create $directoryPath directory" - $displayMsg = "An error occurred when trying to create a new directory" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate - } - $fileName = $entry.FileName.Split([IO.Path]::GetInvalidFileNameChars()) -join '' - $filePath = "$directoryPath/$fileName" - $fileAlreadyExists = Test-Path -Path $filePath -PathType Leaf - if (!$fileAlreadyExists -or $overwrite) { - $entryExportParameters = @{ - Method = "GET" - Uri = "https://$($manifestJson.SubDomain).$($manifestJson.HostName)$($entry.Url)" - Headers = $header - ContentType = "application/json" - } - - try { - if (!$entry.fileName.Contains(".json")) { - Invoke-RestMethod @entryExportParameters -OutFile $filePath | Out-Null - } - else { - $GetEntriesResponse = Invoke-RestMethod @entryExportParameters - $GetEntriesResponse | ConvertTo-Json -Depth 100 | Out-File -FilePath $filePath - } - } - catch { - $message = "Error occurred when extracting data from $($entry.Url)" - Log -msg "An error occurred" -displayMsg $message -logLevel "error" -currentDate $currentDate - Log -msg $message -logLevel "error" -currentDate $currentDate - Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate - Log -msg "Url: $($entry.Url)" -logLevel "error" -currentDate $currentDate - Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate - Log -msg "Exception: $($_.Exception)" -logLevel "error" -currentDate $currentDate - Write-Verbose $_.Exception - Write-Verbose $_.Exception.Response - if ($_.Exception.Response.StatusCode -eq 503) { - $message = "Lost connection to server. Waiting to restablish connection." - Write-ColorOutput red $message - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - do { - $response = ServerHealtCheck("https://$($manifestJson.SubDomain).$($manifestJson.HostName)/en/healthcheck"); - - } while ($response.Exception) - $message = "Server connection reestablished. Resuming export" - Write-ColorOutput green $message - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - $i = $i - 1; - } - } - } - else { - $message = "Skipping $filePath because file already exists" - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - } - } - $msg = "Exporting Instance run finished." - Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate - - try { - Logout($header) - } - catch { - $message = "Error occurred when logging out" - Log -msg $message -displayMsg $message -logLevel "error" -currentDate $currentDate - } - } - else { - $msg = "Exporting Instance run finished with errors." - $displayMsg = "Exporting Instance run was not successful. Check the log file for more details." - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate - } + ExportInstanceData + + $msg = "Exporting Instance run finished." + Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate } else { $manifestFilePath = $manifestFilePath.TrimEnd('\') # Trim any trailing backslash if it exists @@ -354,139 +358,27 @@ This function exports an given instancce and save it to your disk in the specifi $manifestJson = Get-Content -Raw -Path $manifestFile | ConvertFrom-Json $instance = $manifestJson.subdomain - - try { - CreateDirectoryIfDoesNotExist -directoryPath "$exportFilePath\$instance" -currentDate $currentDate - } - catch { - $msg = "Error trying to create $directoryPath directory" - $displayMsg = "An error occurred when trying to create a new directory" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate - Exit 1 - } - - try { - $accessTokenResponseModel = Login - } - catch { - Log -msg "An error occurred" -displayMsg "$($_.Exception.Response.StatusCode.value__): An error occurred when logging in" -logLevel "error" -currentDate $currentDate - Log -msg "Authorization error for Instance Export" -logLevel "error" -currentDate $currentDate - Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate - Log -msg "Url: api/login" -logLevel "error" -currentDate $currentDate - Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate - } - - if ($accessTokenResponseModel) { - - $accessToken = $accessTokenResponseModel.access_token - - $header = @{ - "authorization" = "Bearer $accessToken" - } - - $message = "Starting exporting data for instance $instance" - Log -msg $message -displayMsg $message -logLevel "success" -currentDate $currentDate - - $currentCategory = ""; - $i = 0; - for (; $i -lt $manifestJson.Entries.Length; $i = $i + 1) { - $entry = $manifestJson.Entries[$i] - if ($currentCategory -ne $entry.Category) { - $currentCategory = $entry.Category; - $message = "Starting extraction of $($entry.Category) category." - Log -msg $message -displayMsg $message -logLevel "displayInfo" -currentDate $currentDate - } - $message = "Extracting data from $($entry.Url)" - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - $directoryPath = $exportFilePath + "/" + $instance + "/" + $entry.Path - - try { - CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate - } - catch { - $msg = "Error trying to create $directoryPath directory" - $displayMsg = "An error occurred when trying to create a new directory" - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate - } - $fileName = $entry.FileName.Split([IO.Path]::GetInvalidFileNameChars()) -join '' - $filePath = "$directoryPath/$fileName" - $fileAlreadyExists = Test-Path -Path $filePath -PathType Leaf - if (!$fileAlreadyExists -or $overwrite) { - $entryExportParameters = @{ - Method = "GET" - Uri = "https://$($manifestJson.SubDomain).$($manifestJson.HostName)$($entry.Url)" - Headers = $header - ContentType = "application/json" - } - - try { - if (!$entry.fileName.Contains(".json")) { - Invoke-RestMethod @entryExportParameters -OutFile $filePath | Out-Null - } - else { - $GetEntriesResponse = Invoke-RestMethod @entryExportParameters - $GetEntriesResponse | ConvertTo-Json -Depth 100 | Out-File -FilePath $filePath - } - } - catch { - $message = "Error occurred when extracting data from $($entry.Url)" - Log -msg "An error occurred" -displayMsg $message -logLevel "error" -currentDate $currentDate - Log -msg $message -logLevel "error" -currentDate $currentDate - Log -msg "StatusCode: $($_.Exception.Response.StatusCode.value__)" -logLevel "error" -currentDate $currentDate - Log -msg "Url: $($entry.Url)" -logLevel "error" -currentDate $currentDate - Log -msg "StatusDescription: $($_.Exception.Response.StatusDescription)" -logLevel "error" -currentDate $currentDate - Log -msg "Exception: $($_.Exception)" -logLevel "error" -currentDate $currentDate - Write-Verbose $_.Exception - Write-Verbose $_.Exception.Response - if ($_.Exception.Response.StatusCode -eq 503) { - $message = "Lost connection to server. Waiting to restablish connection." - Write-ColorOutput red $message - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - do { - $response = ServerHealtCheck("https://$($manifestJson.SubDomain).$($manifestJson.HostName)/en/healthcheck"); - } while ($response.Exception) - $message = "Server connection reestablished. Resuming export" - Write-ColorOutput green $message - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - $i = $i - 1; - } - } - } - else { - $message = "Skipping $filePath because file already exists" - Log -msg $message -displayMsg $message -logLevel "info" -currentDate $currentDate - } - } - $msg = "Exporting data for instance $instance was succesful." - Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate - - try { - Logout($header) - } - catch { - $message = "Error occurred when logging out" - Log -msg $message -displayMsg $message -logLevel "error" -currentDate $currentDate - } - } - else { - $msg = "Exporting data for instance $instance finished with errors." - $displayMsg = "Exporting data for instance $instance was not successful. Check the log file for more details." - Log -msg $msg -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate - } + + $directoryPath = $exportFilePath + "/" + $instance + + ExportInstanceData } else { $displayMsg = "An error occurred when looking for manifest file" Log -msg "Error in finding manifest file" -displayMsg $displayMsg -logLevel "error" -currentDate $currentDate } } - } + + $msg = "Exporting Instance script run finished." + Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate + } else { $msg = "Manifest file path doesn't contain valid json file(s)." Log -msg $msg -displayMsg $msg -logLevel "error" -currentDate $currentDate + + $msg = "Exporting Instance script run finished with errors." + Log -msg $msg -displayMsg $msg -logLevel "error" -currentDate $currentDate } - - $msg = "Exporting Instance script run finished." - Log -msg $msg -displayMsg $msg -logLevel "success" -currentDate $currentDate } } else { From 21f02b064c554d5655f9ce1750e557d0cc4999fe Mon Sep 17 00:00:00 2001 From: Nathan Gerhart Date: Wed, 16 Apr 2025 10:55:10 -0600 Subject: [PATCH 7/7] InstanceExport: log start message sooner This way, if a user can't sign in to one of several instances, the logs will clearly show which instance was the problem --- InstanceExport/PowerShell/InstanceExport.ps1 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/InstanceExport/PowerShell/InstanceExport.ps1 b/InstanceExport/PowerShell/InstanceExport.ps1 index 4879565..420a3a2 100644 --- a/InstanceExport/PowerShell/InstanceExport.ps1 +++ b/InstanceExport/PowerShell/InstanceExport.ps1 @@ -181,6 +181,10 @@ Function ServerHealtCheck($Uri) { } Function ExportInstanceData() { + + $message = "Starting exporting data for instance $instance" + Log -msg $message -displayMsg $message -logLevel "success" -currentDate $currentDate + try { CreateDirectoryIfDoesNotExist -directoryPath $directoryPath -currentDate $currentDate } @@ -209,9 +213,6 @@ Function ExportInstanceData() { "authorization" = "Bearer $accessToken" } - $message = "Starting exporting data for instance $instance" - Log -msg $message -displayMsg $message -logLevel "success" -currentDate $currentDate - $currentCategory = ""; $i = 0; for (; $i -lt $manifestJson.Entries.Length; $i = $i + 1) {