From 5763434b7b33c30855b9e66e360cddf8c403e912 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 19 Sep 2025 12:02:47 +0000 Subject: [PATCH 01/13] Update --- .../functions/Get-AzOpsPolicyExemption.ps1 | 6 +- .../functions/Get-AzOpsResourceDefinition.ps1 | 136 +++++++++--------- .../functions/Search-AzOpsAzGraph.ps1 | 30 +++- src/localized/en-us/Strings.psd1 | 6 +- 4 files changed, 102 insertions(+), 76 deletions(-) diff --git a/src/internal/functions/Get-AzOpsPolicyExemption.ps1 b/src/internal/functions/Get-AzOpsPolicyExemption.ps1 index 4d2a7c81..c2fb8426 100644 --- a/src/internal/functions/Get-AzOpsPolicyExemption.ps1 +++ b/src/internal/functions/Get-AzOpsPolicyExemption.ps1 @@ -48,17 +48,17 @@ } if ($Subscription) { if ($SubscriptionsToIncludeResourceGroups -and $ResourceGroup) { - Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsPolicyExemption.Subscription' -LogStringValues $ScopeObject.SubscriptionDisplayName, $ScopeObject.Subscription -Target $ScopeObject + Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsPolicyExemption.Subscription' -LogStringValues $SubscriptionsToIncludeResourceGroups.count -Target $ScopeObject $query = "policyresources | where type == 'microsoft.authorization/policyexemptions' and resourceGroup != '' | order by ['id'] asc" Search-AzOpsAzGraph -Subscription $SubscriptionsToIncludeResourceGroups -Query $query -ErrorAction Stop } elseif ($ResourceGroup) { - Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsPolicyExemption.ResourceGroup' -LogStringValues $ScopeObject.ResourceGroup -Target $ScopeObject + Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsPolicyExemption.ResourceGroup' -LogStringValues $Subscription.count -Target $ScopeObject $query = "policyresources | where type == 'microsoft.authorization/policyexemptions' and resourceGroup != '' | order by ['id'] asc" Search-AzOpsAzGraph -Subscription $Subscription -Query $query -ErrorAction Stop } else { - Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsPolicyExemption.Subscription' -LogStringValues $ScopeObject.SubscriptionDisplayName, $ScopeObject.Subscription -Target $ScopeObject + Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsPolicyExemption.Subscription' -LogStringValues $Subscription.count -Target $ScopeObject $query = "policyresources | where type == 'microsoft.authorization/policyexemptions' and resourceGroup == '' | order by ['id'] asc" Search-AzOpsAzGraph -Subscription $Subscription -Query $query -ErrorAction Stop } diff --git a/src/internal/functions/Get-AzOpsResourceDefinition.ps1 b/src/internal/functions/Get-AzOpsResourceDefinition.ps1 index 107409b4..a60a6823 100644 --- a/src/internal/functions/Get-AzOpsResourceDefinition.ps1 +++ b/src/internal/functions/Get-AzOpsResourceDefinition.ps1 @@ -306,7 +306,7 @@ $resourcesBase = Search-AzOpsAzGraph -Subscription $subscriptions -Query $query -ErrorAction Stop } catch { - Write-AzOpsMessage -LogLevel Warning -LogString 'Get-AzOpsResourceDefinition.Processing.Resource.Warning' -LogStringValues $scopeObject.Name -Target $ScopeObject + Write-AzOpsMessage -LogLevel Warning -LogString 'Get-AzOpsResourceDefinition.Processing.Resource.Warning' -LogStringValues $scopeObject.Name, $_ -Target $ScopeObject } if ($resourcesBase) { $resources = @() @@ -331,84 +331,86 @@ Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsResourceDefinition.SkippingResources' -Target $ScopeObject } # Process Child resources at resource scope in parallel - if (-not $SkipResource -and -not $SkipChildResource) { - if ($SubscriptionsToIncludeChildResource -ne '*') { - $resources = $resources | Where-Object { $_.subscriptionId -in $SubscriptionsToIncludeChildResource } - } - $resources | Foreach-Object -ThrottleLimit (Get-PSFConfigValue -FullName 'AzOps.Core.ThrottleLimit') -Parallel { - $resource = $_ - $runspaceData = $using:runspaceData + if ($resources.Count -gt 0) { + if (-not $SkipResource -and -not $SkipChildResource) { + if ($SubscriptionsToIncludeChildResource -ne '*') { + $resources = $resources | Where-Object { $_.subscriptionId -in $SubscriptionsToIncludeChildResource } + } + $resources | Foreach-Object -ThrottleLimit (Get-PSFConfigValue -FullName 'AzOps.Core.ThrottleLimit') -Parallel { + $resource = $_ + $runspaceData = $using:runspaceData - Import-Module "$([PSFramework.PSFCore.PSFCoreHost]::ModuleRoot)/PSFramework.psd1" - $azOps = Import-Module $runspaceData.AzOpsPath -Force -PassThru + Import-Module "$([PSFramework.PSFCore.PSFCoreHost]::ModuleRoot)/PSFramework.psd1" + $azOps = Import-Module $runspaceData.AzOpsPath -Force -PassThru - & $azOps { - $script:AzOpsAzManagementGroup = $runspaceData.runspace_AzOpsAzManagementGroup - $script:AzOpsSubscriptions = $runspaceData.runspace_AzOpsSubscriptions - $script:AzOpsPartialRoot = $runspaceData.runspace_AzOpsPartialRoot - $script:AzOpsResourceProvider = $runspaceData.runspace_AzOpsResourceProvider - $script:AzOpsGraphResourceProvider = $runspaceData.runspace_AzOpsGraphResourceProvider - } - $context = Get-AzContext - $context.Subscription.Id = $resource.subscriptionId - $tempExportPath = [System.IO.Path]::GetTempPath() + (New-Guid).ToString() + '.json' - try { & $azOps { - # Validate resource group name before calling Export-AzResourceGroup - if ([string]::IsNullOrEmpty($resource.resourceGroup)) { - Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsResourceDefinition.Processing.ChildResource.SkippingNoResourceGroup' -LogStringValues $resource.name, $resource.id -Target $resource - return - } - $exportParameters = @{ - Resource = $resource.id - ResourceGroupName = $resource.resourceGroup - SkipAllParameterization = $true - Path = $tempExportPath - DefaultProfile = $context | Select-Object -First 1 - } - Invoke-AzOpsScriptBlock -ArgumentList $exportParameters -ScriptBlock { - param ( - $ExportParameters - ) - $param = $ExportParameters | Write-Output - Export-AzResourceGroup @param -Confirm:$false -Force -ErrorAction Stop | Out-Null - } -RetryCount $runspaceData.MaxRetryCount -RetryWait $runspaceData.BackoffMultiplier -RetryType Exponential + $script:AzOpsAzManagementGroup = $runspaceData.runspace_AzOpsAzManagementGroup + $script:AzOpsSubscriptions = $runspaceData.runspace_AzOpsSubscriptions + $script:AzOpsPartialRoot = $runspaceData.runspace_AzOpsPartialRoot + $script:AzOpsResourceProvider = $runspaceData.runspace_AzOpsResourceProvider + $script:AzOpsGraphResourceProvider = $runspaceData.runspace_AzOpsGraphResourceProvider } - $exportResources = (Get-Content -Path $tempExportPath | ConvertFrom-Json).resources - $resourceGroup = $using:resourceGroups | Where-Object {$_.subscriptionId -eq $resource.subscriptionId -and $_.name -eq $resource.resourceGroup} - foreach ($exportResource in $exportResources) { - if (-not(($resource.name -eq $exportResource.name) -and ($resource.type -eq $exportResource.type))) { - & $azOps { - Write-AzOpsMessage -LogLevel Verbose -LogString 'Get-AzOpsResourceDefinition.Processing.ChildResource' -LogStringValues $exportResource.name, $resource.resourceGroup -FunctionName "Get-AzOpsResourceDefinition" -ModuleName "AzOps" -Target $exportResource - } - $ChildResource = @{ - resourceProvider = $exportResource.type -replace '/', '_' - resourceName = $exportResource.name -replace '/', '_' - parentResourceId = $resourceGroup.id + $context = Get-AzContext + $context.Subscription.Id = $resource.subscriptionId + $tempExportPath = [System.IO.Path]::GetTempPath() + (New-Guid).ToString() + '.json' + try { + & $azOps { + # Validate resource group name before calling Export-AzResourceGroup + if ([string]::IsNullOrEmpty($resource.resourceGroup)) { + Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsResourceDefinition.Processing.ChildResource.SkippingNoResourceGroup' -LogStringValues $resource.name, $resource.id -Target $resource + return } - if (Get-Member -InputObject $exportResource -name 'dependsOn') { - $exportResource.PsObject.Members.Remove('dependsOn') + $exportParameters = @{ + Resource = $resource.id + ResourceGroupName = $resource.resourceGroup + SkipAllParameterization = $true + Path = $tempExportPath + DefaultProfile = $context | Select-Object -First 1 } - $resourceHash = @{resources = @($exportResource) } - & $azOps { - ConvertTo-AzOpsState -Resource $resourceHash -ChildResource $ChildResource -StatePath $runspaceData.Statepath + Invoke-AzOpsScriptBlock -ArgumentList $exportParameters -ScriptBlock { + param ( + $ExportParameters + ) + $param = $ExportParameters | Write-Output + Export-AzResourceGroup @param -Confirm:$false -Force -ErrorAction Stop | Out-Null + } -RetryCount $runspaceData.MaxRetryCount -RetryWait $runspaceData.BackoffMultiplier -RetryType Exponential + } + $exportResources = (Get-Content -Path $tempExportPath | ConvertFrom-Json).resources + $resourceGroup = $using:resourceGroups | Where-Object {$_.subscriptionId -eq $resource.subscriptionId -and $_.name -eq $resource.resourceGroup} + foreach ($exportResource in $exportResources) { + if (-not(($resource.name -eq $exportResource.name) -and ($resource.type -eq $exportResource.type))) { + & $azOps { + Write-AzOpsMessage -LogLevel Verbose -LogString 'Get-AzOpsResourceDefinition.Processing.ChildResource' -LogStringValues $exportResource.name, $resource.resourceGroup -FunctionName "Get-AzOpsResourceDefinition" -ModuleName "AzOps" -Target $exportResource + } + $ChildResource = @{ + resourceProvider = $exportResource.type -replace '/', '_' + resourceName = $exportResource.name -replace '/', '_' + parentResourceId = $resourceGroup.id + } + if (Get-Member -InputObject $exportResource -name 'dependsOn') { + $exportResource.PsObject.Members.Remove('dependsOn') + } + $resourceHash = @{resources = @($exportResource) } + & $azOps { + ConvertTo-AzOpsState -Resource $resourceHash -ChildResource $ChildResource -StatePath $runspaceData.Statepath + } } } } - } - catch { - & $azOps { - Write-AzOpsMessage -LogLevel Warning -LogString 'Get-AzOpsResourceDefinition.ChildResource.Warning' -LogStringValues $resource.resourceGroup, ($exportParameters | Out-String -NoNewline), $_ -FunctionName "Get-AzOpsResourceDefinition" -ModuleName "AzOps" + catch { + & $azOps { + Write-AzOpsMessage -LogLevel Warning -LogString 'Get-AzOpsResourceDefinition.ChildResource.Warning' -LogStringValues $resource.resourceGroup, ($exportParameters | Out-String -NoNewline), $_ -FunctionName "Get-AzOpsResourceDefinition" -ModuleName "AzOps" + } + } + if (Test-Path -Path $tempExportPath) { + Remove-Item -Path $tempExportPath -Force } } - if (Test-Path -Path $tempExportPath) { - Remove-Item -Path $tempExportPath -Force - } + Clear-PSFMessage + } + else { + Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsResourceDefinition.SkippingChildResources' -Target $ScopeObject } - Clear-PSFMessage - } - else { - Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsResourceDefinition.SkippingChildResources' -Target $ScopeObject } } #endregion Process Resource Groups diff --git a/src/internal/functions/Search-AzOpsAzGraph.ps1 b/src/internal/functions/Search-AzOpsAzGraph.ps1 index 9a25f719..3db072d1 100644 --- a/src/internal/functions/Search-AzOpsAzGraph.ps1 +++ b/src/internal/functions/Search-AzOpsAzGraph.ps1 @@ -7,9 +7,16 @@ .PARAMETER UseTenantScope Use Tenant as Scope true or false .PARAMETER ManagementGroupName - ManagementGroup Name + ManagementGroup Id .PARAMETER Subscription - Subscription Id's + Subscription object(s) containing subscription information. Can be a single object or array of objects. + Each object must have an 'Id' property with a valid GUID. + Example structure: + @{ + "Name" = "MySubscription" + "Id" = "1ea96474-9e13-442f-afe3-b2e7810e6rb8" + "Type" = "/subscriptions" + } .PARAMETER Query AzureResourceGraph-Query .EXAMPLE @@ -23,9 +30,26 @@ [switch] $UseTenantScope, [Parameter(Mandatory = $false)] - [string] + [guid] $ManagementGroupName, [Parameter(Mandatory = $false)] + [ValidateScript({ + # Allow null input + if ($null -eq $_) { return $true } + # Convert single object to array for uniform processing + $subscriptions = if ($_ -is [array]) { $_ } else { @($_) } + foreach ($sub in $subscriptions) { + # Validate Id property exists + if (-not ($sub.PSObject.Properties.Name -contains 'Id')) { + throw "Subscription Id is missing: [$sub]" + } + # Validate Id is a valid GUID + if (-not ($sub.Id -match '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$')) { + throw "Subscription Id must be a valid GUID format: [$sub]" + } + } + return $true + })] [object] $Subscription, [Parameter(Mandatory = $true, ValueFromPipeline = $true)] diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index 4205cced..b4ed958f 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -110,8 +110,8 @@ 'Get-AzOpsPolicyDefinition.Subscription' = 'Retrieving custom policy definitions for {0} Subscription objects' # $Subscription.count 'Get-AzOpsPolicyExemption.ManagementGroup' = 'Retrieving Policy Exemption for Management Group {0} ({1})' # $ScopeObject.ManagementGroupDisplayName, $ScopeObject.ManagementGroup - 'Get-AzOpsPolicyExemption.ResourceGroup' = 'Retrieving Policy Exemption for Resource Group {0}' # $ScopeObject.ResourceGroup - 'Get-AzOpsPolicyExemption.Subscription' = 'Retrieving Policy Exemption for Subscription {0} ({1})' # $ScopeObject.SubscriptionDisplayName, $ScopeObject.Subscription + 'Get-AzOpsPolicyExemption.ResourceGroup' = 'Retrieving Policy Exemption for Resource Group in {0} Subscription objects' # $Subscription.count + 'Get-AzOpsPolicyExemption.Subscription' = 'Retrieving Policy Exemption for {0} Subscription objects' # $Subscription.count 'Get-AzOpsResource.Failed' = 'Failed retrieving resource with error: {0}' # $_ @@ -135,7 +135,7 @@ 'Get-AzOpsResourceDefinition.Processing.Resource.Discovery' = 'Searching for resources in [{0}]' # $scopeObject.Name 'Get-AzOpsResourceDefinition.Processing.Resource.Discovery.NotFound' = 'No resources found in [{0}]' # $scopeObject.Name 'Get-AzOpsResourceDefinition.Processing.Resource.SkippingNoResourceGroup' = 'Skipping resource [{0}] at [{1}], null or empty ResourceGroup property. Cannot export resources' # $resource.name, $resource.id - 'Get-AzOpsResourceDefinition.Processing.Resource.Warning' = 'Failed to get resources in [{0}]. Consider excluding the resource causing the failure with [Core.SkipResourceType] setting' # $scopeObject.Name + 'Get-AzOpsResourceDefinition.Processing.Resource.Warning' = 'Failed to get resources in [{0}]. Consider excluding the resource causing the failure with [Core.SkipResourceType] setting. Error message: [{1}]' # $scopeObject.Name, $_ 'Get-AzOpsResourceDefinition.SkippingResourceGroup' = 'SkipResourceGroup switch used, skipping resource Group discovery' # 'Get-AzOpsResourceDefinition.SkippingResources' = 'SkipResource switch used, skipping resource discovery.' # 'Get-AzOpsResourceDefinition.Processing.ChildResource' = 'Processing resource [{0}] in resource Group [{1}]' # $resource.Name, $resourceGroup.ResourceGroupName From 405702136d01681a229cee9e11aac47c0a7984f8 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 1 Oct 2025 10:12:40 +0000 Subject: [PATCH 02/13] Update --- src/internal/functions/Search-AzOpsAzGraph.ps1 | 5 +++++ src/localized/en-us/Strings.psd1 | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/internal/functions/Search-AzOpsAzGraph.ps1 b/src/internal/functions/Search-AzOpsAzGraph.ps1 index 3db072d1..7b8f23bd 100644 --- a/src/internal/functions/Search-AzOpsAzGraph.ps1 +++ b/src/internal/functions/Search-AzOpsAzGraph.ps1 @@ -61,6 +61,7 @@ Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing' -LogStringValues $Query $results = @() if ($UseTenantScope) { + Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.UseTenantScope' do { $processing = Search-AzGraph -UseTenantScope -Query $Query -AllowPartialScope -SkipToken $processing.SkipToken -ErrorAction Stop $results += $processing @@ -68,6 +69,7 @@ while ($processing.SkipToken) } if ($ManagementGroupName) { + Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.ManagementGroup' -LogStringValues $ManagementGroupName do { $processing = Search-AzGraph -ManagementGroup $ManagementGroupName -Query $Query -AllowPartialScope -SkipToken $processing.SkipToken -ErrorAction Stop $results += $processing @@ -81,6 +83,9 @@ # Group subscriptions into batches to conform with graph limits $subscriptionBatch = $Subscription | Group-Object -Property { [math]::Floor($counter.Value++ / $batchSize) } foreach ($group in $subscriptionBatch) { + $subscriptionIds = ($group.Group).Id -join ', ' + $subscriptionCount = $group.Group.Count + Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.SubscriptionBatch' -LogStringValues $subscriptionCount, $subscriptionIds do { $processing = Search-AzGraph -Subscription ($group.Group).Id -Query $Query -SkipToken $processing.SkipToken -ErrorAction Stop $results += $processing diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index b4ed958f..d5fd84fb 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -345,6 +345,9 @@ 'Save-AzOpsManagementGroupChild.Subscription.NotFound' = 'Unable to locate subscription: {0} within AzOpsSubscriptions object' #child.Name 'Search-AzOpsAzGraph.Processing' = 'AzGraph processing query: [{0}]' # $Query + 'Search-AzOpsAzGraph.Processing.UseTenantScope' = 'AzGraph processing query at tenantScope [/]' # + 'Search-AzOpsAzGraph.Processing.ManagementGroup' = 'AzGraph processing query at managementGroup: [{0}]' # $ManagementGroupName + 'Search-AzOpsAzGraph.Processing.SubscriptionBatch' = 'AzGraph processing query at {0} subscriptions: [{1}]' # $subscriptionCount, $subscriptionIds 'Search-AzOpsAzGraph.Processing.Done' = 'AzGraph completed processing of query: [{0}]' # $Query 'Search-AzOpsAzGraph.Processing.NoResult' = 'AzGraph found nothing with query: [{0}]' # $Query From 28fd1895407382a55fefe5eabee908c769a52c70 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Mon, 6 Oct 2025 15:20:44 +0000 Subject: [PATCH 03/13] Update --- .../functions/Search-AzOpsAzGraph.ps1 | 159 ++++++++++++++---- src/localized/en-us/Strings.psd1 | 12 ++ 2 files changed, 142 insertions(+), 29 deletions(-) diff --git a/src/internal/functions/Search-AzOpsAzGraph.ps1 b/src/internal/functions/Search-AzOpsAzGraph.ps1 index 7b8f23bd..9f13fd86 100644 --- a/src/internal/functions/Search-AzOpsAzGraph.ps1 +++ b/src/internal/functions/Search-AzOpsAzGraph.ps1 @@ -59,58 +59,159 @@ process { Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing' -LogStringValues $Query - $results = @() + $results = [System.Collections.Generic.List[object]]::new() + if ($UseTenantScope) { Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.UseTenantScope' - do { - $processing = Search-AzGraph -UseTenantScope -Query $Query -AllowPartialScope -SkipToken $processing.SkipToken -ErrorAction Stop - $results += $processing + try { + do { + $tenantProcessing = Search-AzGraph -UseTenantScope -Query $Query -AllowPartialScope -SkipToken $tenantProcessing.SkipToken -ErrorAction Stop + if ($tenantProcessing) { $results.AddRange($tenantProcessing) } + } + while ($tenantProcessing.SkipToken) + } + catch { + Write-AzOpsMessage -LogLevel Error -LogString 'Search-AzOpsAzGraph.Processing.UseTenantScope.Failed' -LogStringValues $Query, $_.Exception.Message } - while ($processing.SkipToken) } + if ($ManagementGroupName) { Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.ManagementGroup' -LogStringValues $ManagementGroupName - do { - $processing = Search-AzGraph -ManagementGroup $ManagementGroupName -Query $Query -AllowPartialScope -SkipToken $processing.SkipToken -ErrorAction Stop - $results += $processing + try { + do { + $mgProcessing = Search-AzGraph -ManagementGroup $ManagementGroupName -Query $Query -AllowPartialScope -SkipToken $mgProcessing.SkipToken -ErrorAction Stop + if ($mgProcessing) { $results.AddRange($mgProcessing) } + } + while ($mgProcessing.SkipToken) + } + catch { + Write-AzOpsMessage -LogLevel Error -LogString 'Search-AzOpsAzGraph.Processing.ManagementGroup.Failed' -LogStringValues $Query, $ManagementGroupName, $_.Exception.Message } - while ($processing.SkipToken) } + if ($Subscription) { # Create a counter, set the batch size, and prepare a variable for the results - $counter = [PSCustomObject] @{ Value = 0 } + $counter = 0 $batchSize = 1000 # Group subscriptions into batches to conform with graph limits - $subscriptionBatch = $Subscription | Group-Object -Property { [math]::Floor($counter.Value++ / $batchSize) } + $subscriptionBatch = $Subscription | Group-Object -Property { [math]::Floor($counter++ / $batchSize) } + foreach ($group in $subscriptionBatch) { $subscriptionIds = ($group.Group).Id -join ', ' $subscriptionCount = $group.Group.Count Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.SubscriptionBatch' -LogStringValues $subscriptionCount, $subscriptionIds - do { - $processing = Search-AzGraph -Subscription ($group.Group).Id -Query $Query -SkipToken $processing.SkipToken -ErrorAction Stop - $results += $processing + + try { + $batchProcessing = $null + do { + $batchProcessing = Search-AzGraph -Subscription ($group.Group).Id -Query $Query -SkipToken $batchProcessing.SkipToken -ErrorAction Stop + if ($batchProcessing) { $results.AddRange($batchProcessing) } + } + while ($batchProcessing.SkipToken) + } + catch { + # Batch failed - try each subscription individually to identify the problematic scope + Write-AzOpsMessage -LogLevel Warning -LogString 'Search-AzOpsAzGraph.Processing.SubscriptionBatch.Failed' -LogStringValues $subscriptionIds, $_.Exception.Message + Write-AzOpsMessage -LogLevel Important -LogString 'Search-AzOpsAzGraph.Processing.SubscriptionBatch.RetryIndividually' -LogStringValues $subscriptionCount + + foreach ($sub in $group.Group) { + try { + Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.Subscription' -LogStringValues $sub.Name, $sub.Id + $subProcessing = $null + do { + $subProcessing = Search-AzGraph -Subscription $sub.Id -Query $Query -SkipToken $subProcessing.SkipToken -ErrorAction Stop + if ($subProcessing) { $results.AddRange($subProcessing) } + } + while ($subProcessing.SkipToken) + } + catch { + Write-AzOpsMessage -LogLevel Error -LogString 'Search-AzOpsAzGraph.Processing.Subscription.Failed' -LogStringValues $Query, $sub.Name, $sub.Id, $_.Exception.Message + + try { + Write-AzOpsMessage -LogLevel Debug -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RetryWithRestApi' -LogStringValues $sub.Id + $resourceGraphApiVersion = (($script:AzOpsResourceProvider | Where-Object {$_.ProviderNamespace -eq 'Microsoft.ResourceGraph'}).ResourceTypes | Where-Object {$_.ResourceTypeName -eq 'queries'}).ApiVersions | Select-Object -First 1 + # Initialize request body for first call + $requestBody = @{ + subscriptions = @($sub.Id) + query = $Query + } | ConvertTo-Json -Depth 10 + + $restApiResponse = $null + do { + $response = Invoke-AzRestMethod -Method POST -Path "/providers/Microsoft.ResourceGraph/resources?api-version=$resourceGraphApiVersion" -Payload $requestBody -ErrorAction Stop + + if ($response.StatusCode -eq 200) { + $restApiResponse = $response.Content | ConvertFrom-Json -Depth 100 -ErrorAction Stop + + if ($restApiResponse.data) { + Write-AzOpsMessage -LogLevel Important -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RestApiSuccess' -LogStringValues $sub.Id, $restApiResponse.data.Count + $results.AddRange($restApiResponse.data) + } + + # Prepare next page request if skipToken exists + if ($restApiResponse.'$skipToken') { + $requestBody = @{ + subscriptions = @($sub.Id) + query = $Query + options = @{ + '$skipToken' = $restApiResponse.'$skipToken' + } + } | ConvertTo-Json -Depth 10 + } + } + else { + # Log the raw error response for analysis + Write-AzOpsMessage -LogLevel Error -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RestApiFailed' -LogStringValues $sub.Id, $response.StatusCode, $response.Content + + # Attempt to parse error details + try { + $errorContent = $response.Content | ConvertFrom-Json -ErrorAction Stop + if ($errorContent.error) { + Write-AzOpsMessage -LogLevel Error -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RestApiErrorDetails' -LogStringValues $errorContent.error.code, $errorContent.error.message + } + } + catch { + Write-AzOpsMessage -LogLevel Debug -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RestApiRawError' -LogStringValues $response.Content + } + # Break pagination loop on error + break + } + } while ($restApiResponse.'$skipToken') + } + catch { + # Log REST API fallback error but continue processing other subscriptions + Write-AzOpsMessage -LogLevel Error -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RestApiException' -LogStringValues $Query, $sub.Id, $_.Exception.Message + } + # Continue processing remaining subscriptions in the group + } + } } - while ($processing.SkipToken) } } + if ($results) { - $resultsType = @() - foreach ($result in $results) { - # Process each graph result and normalize ProviderNamespace casing - foreach ($ResourceProvider in $script:AzOpsResourceProvider) { - if ($ResourceProvider.ProviderNamespace -eq $result.type.Split('/')[0]) { - foreach ($ResourceTypeName in $ResourceProvider.ResourceTypes.ResourceTypeName) { - if ($ResourceTypeName -eq $result.type.Split('/')[1]) { - $result.type = ($result.type).replace($result.type.Split('/')[0],$ResourceProvider.ProviderNamespace) - $result.type = ($result.type).replace($result.type.Split('/')[1],$ResourceTypeName) - $resultsType += $result - break - } - } - break + $providerLookup = @{} + foreach ($ResourceProvider in $script:AzOpsResourceProvider) { + foreach ($ResourceTypeName in $ResourceProvider.ResourceTypes.ResourceTypeName) { + # Use lowercase key for case-insensitive matching + $key = "$($ResourceProvider.ProviderNamespace)/$ResourceTypeName".ToLower() + $providerLookup[$key] = @{ + Namespace = $ResourceProvider.ProviderNamespace + TypeName = $ResourceTypeName } } } + + $resultsType = [System.Collections.Generic.List[object]]::new() + foreach ($result in $results) { + # Process each graph result and normalize ProviderNamespace casing using hashtable lookup + $resultTypeKey = $result.type.ToLower() + if ($providerLookup.ContainsKey($resultTypeKey)) { + # Reconstruct the type with correct casing from the lookup + $result.type = "$($providerLookup[$resultTypeKey].Namespace)/$($providerLookup[$resultTypeKey].TypeName)" + $resultsType.Add($result) + } + } Write-AzOpsMessage -LogLevel Debug -LogString 'Search-AzOpsAzGraph.Processing.Done' -LogStringValues $Query return $resultsType } diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index d5fd84fb..dbcbcca1 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -346,8 +346,20 @@ 'Search-AzOpsAzGraph.Processing' = 'AzGraph processing query: [{0}]' # $Query 'Search-AzOpsAzGraph.Processing.UseTenantScope' = 'AzGraph processing query at tenantScope [/]' # + 'Search-AzOpsAzGraph.Processing.UseTenantScope.Failed' = 'Failed AzGraph processing query: [{0}] at tenantScope [/] with: {1}' # $Query, $_.Exception.Message 'Search-AzOpsAzGraph.Processing.ManagementGroup' = 'AzGraph processing query at managementGroup: [{0}]' # $ManagementGroupName + 'Search-AzOpsAzGraph.Processing.ManagementGroup.Failed' = 'Failed AzGraph processing query: [{0}] at managementGroup: [{1}] with: {2}' # $Query, $ManagementGroupName, $_.Exception.Message + 'Search-AzOpsAzGraph.Processing.Subscription' = 'AzGraph processing query at subscription: [{0}({1})]' # $sub.Name, $sub.Id + 'Search-AzOpsAzGraph.Processing.Subscription.Failed' = 'Failed AzGraph processing query: [{0}] at subscription: [{1}({2})] with: {3}' # $Query, $sub.Name, $sub.Id, $_.Exception.Message + 'Search-AzOpsAzGraph.Processing.Subscription.RetryWithRestApi' = 'Retrying Microsoft.ResourceGraph query at subscription [{0}] using REST API for additional diagnostics' # $sub.Id + 'Search-AzOpsAzGraph.Processing.Subscription.RestApiSuccess' = 'Microsoft.ResourceGraph REST API query succeeded for subscription [{0}] with [{1}] results' # $sub.Id, $responseContent.data.Count + 'Search-AzOpsAzGraph.Processing.Subscription.RestApiFailed' = 'Microsoft.ResourceGraph REST API query failed for subscription [{0}] with status [{1}]: {2}' # $sub.Id, $response.StatusCode, $response.Content + 'Search-AzOpsAzGraph.Processing.Subscription.RestApiErrorDetails' = 'Microsoft.ResourceGraph REST API error code [{0}]: {1}' # $errorContent.error.code, $errorContent.error.message + 'Search-AzOpsAzGraph.Processing.Subscription.RestApiRawError' = 'Microsoft.ResourceGraph REST API raw error response: {0}' # $response.Content + 'Search-AzOpsAzGraph.Processing.Subscription.RestApiException' = 'Microsoft.ResourceGraph REST API exception for query: [{0}] at subscription [{1}]: {2}' # $Query, $sub.Id, $_.Exception.Message 'Search-AzOpsAzGraph.Processing.SubscriptionBatch' = 'AzGraph processing query at {0} subscriptions: [{1}]' # $subscriptionCount, $subscriptionIds + 'Search-AzOpsAzGraph.Processing.SubscriptionBatch.Failed' = 'Failed AzGraph processing query at subscriptions: [{0}] with: {1}' # $subscriptionIds, $_.Exception.Message + 'Search-AzOpsAzGraph.Processing.SubscriptionBatch.RetryIndividually' = 'Retrying AzGraph processing query of {0} subscriptions individually' # $subscriptionCount, $subscriptionIds 'Search-AzOpsAzGraph.Processing.Done' = 'AzGraph completed processing of query: [{0}]' # $Query 'Search-AzOpsAzGraph.Processing.NoResult' = 'AzGraph found nothing with query: [{0}]' # $Query From 8a90703ac9c755611ba38c30e1b25ba4189ad563 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 7 Oct 2025 14:32:24 +0000 Subject: [PATCH 04/13] Update --- docs/wiki/Settings.md | 21 +++---- src/internal/configurations/Core.ps1 | 1 + .../functions/Get-AzOpsResourceDefinition.ps1 | 19 +++++- .../functions/Search-AzOpsAzGraph.ps1 | 59 +++++++++++++++---- src/localized/en-us/Strings.psd1 | 2 + 5 files changed, 78 insertions(+), 24 deletions(-) diff --git a/docs/wiki/Settings.md b/docs/wiki/Settings.md index c6dd2a43..f99179f0 100644 --- a/docs/wiki/Settings.md +++ b/docs/wiki/Settings.md @@ -34,16 +34,17 @@ The following configuration values can be modified within the `settings.json` fi | 22 | SkipResourceGroup | Do not include Resource Groups in pull | `"Core.SkipResourceGroup": false` | | 23 | SkipResourceType | Skip specific [Resource Types](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types) (only targets Resource Group scoped resources) | `"Core.SkipResourceType": ["Microsoft.VSOnline/plans"]` | | 24 | SkipRole | Do not include Role types in pull | `"Core.SkipRole": false` | -| 25 | State | Folder to store AzOpsState artefact, defaults to `root` | `"Core.State: "/root"` | -| 26 | SubscriptionsToIncludeChildResource | Filter which Subscription IDs should include child resources in pull | `"Core.SubscriptionsToIncludeChildResource": ["*"]` | -| 27 | SubscriptionsToIncludeResourceGroups | Filter which Subscription IDs should include Resource Groups in pull [Logic Updated in v2.0.0](https://github.com/Azure/AzOps/releases/tag/2.0.0) | `"Core.SubscriptionsToIncludeResourceGroups": ["*"]` | -| 28 | TemplateParameterFileSuffix | Default template file suffix. *Not recommended to change* | `"Core.TemplateParameterFileSuffix": ".json"` | -| 29 | AllowMultipleTemplateParameterFiles | Control multiple parameter file behaviour. *Not recommended to change* | `"Core.AllowMultipleTemplateParameterFiles": false` | -| 30 | DeployAllMultipleTemplateParameterFiles | Control base template deployment behaviour with changes and un-changed multiple corresponding parameter files. | `"Core.DeployAllMultipleTemplateParameterFiles": false` | -| 31 | MultipleTemplateParameterFileSuffix | Multiple parameter file suffix identifier. *Example mytemplate.x1.bicepparam* | `"Core.MultipleTemplateParameterFileSuffix": ".x"` | -| 32 | ParallelDeployMultipleTemplateParameterFiles | Control parallel deployment of MultipleTemplateParameterFiles behaviour | `"Core.ParallelDeployMultipleTemplateParameterFiles": false` | -| 33 | ThrottleLimit | Value declaring number of parallel threads. [Read more](https://github.com/azure/azops/wiki/performance-considerations) | `"Core.ThrottleLimit": 5` | -| 34 | WhatifExcludedChangeTypes | Exclude specific change types from WhatIf operations | `"Core.WhatifExcludedChangeTypes": ["NoChange","Ignore"]` | +| 25 | SkipSubscription | Filter which Subscription IDs should be excluded from pull | `"Core.SkipSubscription": ["*"]` | +| 26 | State | Folder to store AzOpsState artefact, defaults to `root` | `"Core.State: "/root"` | +| 27 | SubscriptionsToIncludeChildResource | Filter which Subscription IDs should include child resources in pull | `"Core.SubscriptionsToIncludeChildResource": ["*"]` | +| 28 | SubscriptionsToIncludeResourceGroups | Filter which Subscription IDs should include Resource Groups in pull [Logic Updated in v2.0.0](https://github.com/Azure/AzOps/releases/tag/2.0.0) | `"Core.SubscriptionsToIncludeResourceGroups": ["*"]` | +| 29 | TemplateParameterFileSuffix | Default template file suffix. *Not recommended to change* | `"Core.TemplateParameterFileSuffix": ".json"` | +| 30 | AllowMultipleTemplateParameterFiles | Control multiple parameter file behaviour. *Not recommended to change* | `"Core.AllowMultipleTemplateParameterFiles": false` | +| 31 | DeployAllMultipleTemplateParameterFiles | Control base template deployment behaviour with changes and un-changed multiple corresponding parameter files. | `"Core.DeployAllMultipleTemplateParameterFiles": false` | +| 32 | MultipleTemplateParameterFileSuffix | Multiple parameter file suffix identifier. *Example mytemplate.x1.bicepparam* | `"Core.MultipleTemplateParameterFileSuffix": ".x"` | +| 33 | ParallelDeployMultipleTemplateParameterFiles | Control parallel deployment of MultipleTemplateParameterFiles behaviour | `"Core.ParallelDeployMultipleTemplateParameterFiles": false` | +| 34 | ThrottleLimit | Value declaring number of parallel threads. [Read more](https://github.com/azure/azops/wiki/performance-considerations) | `"Core.ThrottleLimit": 5` | +| 35 | WhatifExcludedChangeTypes | Exclude specific change types from WhatIf operations | `"Core.WhatifExcludedChangeTypes": ["NoChange","Ignore"]` | ## Workflow / Pipeline Settings diff --git a/src/internal/configurations/Core.ps1 b/src/internal/configurations/Core.ps1 index 3bb636af..a6dd3936 100644 --- a/src/internal/configurations/Core.ps1 +++ b/src/internal/configurations/Core.ps1 @@ -25,6 +25,7 @@ Set-PSFConfig -Module AzOps -Name Core.SkipResource -Value $true -Initialize -Va Set-PSFConfig -Module AzOps -Name Core.SkipResourceGroup -Value $false -Initialize -Validation bool -Description 'Global flag to indicate whether resource group should be discovered or not' Set-PSFConfig -Module AzOps -Name Core.SkipResourceType -Value @('Microsoft.VSOnline/plans', 'Microsoft.PowerPlatform/accounts', 'Microsoft.PowerPlatform/enterprisePolicies') -Initialize -Validation stringarray -Description 'Global flag to skip discovery of specific Resource types.' Set-PSFConfig -Module AzOps -Name Core.SkipRole -Value $false -Initialize -Validation bool -Description '-' +Set-PSFConfig -Module AzOps -Name Core.SkipSubscription -Value @('*') -Initialize -Validation stringarray -Description 'Skip Subscription ID that matches the filter.' Set-PSFConfig -Module AzOps -Name Core.State -Value (Join-Path $pwd -ChildPath "root") -Initialize -Validation string -Description 'Folder to store AzOpsState artefact' Set-PSFConfig -Module AzOps -Name Core.SubscriptionsToIncludeChildResource -Value @('*') -Initialize -Validation stringarray -Description 'Requires SkipResourceGroup, SkipResource and SkipChildResource to be false. Subscription ID that matches the filter.' Set-PSFConfig -Module AzOps -Name Core.SubscriptionsToIncludeResourceGroups -Value @('*') -Initialize -Validation stringarray -Description 'Requires SkipResourceGroup to be false. Subscription ID that matches the filter.' diff --git a/src/internal/functions/Get-AzOpsResourceDefinition.ps1 b/src/internal/functions/Get-AzOpsResourceDefinition.ps1 index a60a6823..0b483273 100644 --- a/src/internal/functions/Get-AzOpsResourceDefinition.ps1 +++ b/src/internal/functions/Get-AzOpsResourceDefinition.ps1 @@ -27,6 +27,8 @@ Skip discovery of specific resource types. .PARAMETER SkipRole Skip discovery of roles for better performance. + .PARAMETER SkipSubscription + Filter which Subscription IDs should be excluded from pull. .PARAMETER StatePath The root folder under which to write the resource json. .PARAMETER SubscriptionsToIncludeChildResource @@ -86,6 +88,9 @@ [switch] $SkipRole = (Get-PSFConfigValue -FullName 'AzOps.Core.SkipRole'), + [string[]] + $SkipSubscription = (Get-PSFConfigValue -FullName 'AzOps.Core.SkipSubscription'), + [Parameter(Mandatory = $false)] [string] $StatePath = (Get-PSFConfigValue -FullName 'AzOps.Core.State'), @@ -131,13 +136,20 @@ switch ($scopeObject.Type) { subscriptions { Write-AzOpsMessage -LogLevel Important -LogString 'Get-AzOpsResourceDefinition.Subscription.Processing' -LogStringValues $ScopeObject.SubscriptionDisplayName, $ScopeObject.Subscription -Target $ScopeObject - $subscriptions = Get-AzSubscription -SubscriptionId $scopeObject.Name | Where-Object { "/subscriptions/$($_.Id)" -in $script:AzOpsSubscriptions.id } + $subscriptions = Get-AzSubscription -SubscriptionId $scopeObject.Name | Where-Object { + "/subscriptions/$($_.Id)" -in $script:AzOpsSubscriptions.id -and + $_.Id -notin $SkipSubscription + } } managementGroups { Write-AzOpsMessage -LogLevel Important -LogString 'Get-AzOpsResourceDefinition.ManagementGroup.Processing' -LogStringValues $ScopeObject.ManagementGroupDisplayName, $ScopeObject.ManagementGroup -Target $ScopeObject $query = "resourcecontainers | where type == 'microsoft.management/managementgroups' | order by ['id'] asc" $managementgroups = Search-AzOpsAzGraph -ManagementGroupName $scopeObject.Name -Query $query -ErrorAction Stop | Where-Object { $_.id -in $script:AzOpsAzManagementGroup.Id } $subscriptions = Get-AzOpsNestedSubscription -Scope $scopeObject.Name + # Filter out skipped subscriptions + if ($SkipSubscription) { + $subscriptions = $subscriptions | Where-Object { $_.Id -notin $SkipSubscription } + } if ($managementgroups) { # Process managementGroup scope in parallel $managementgroups | Foreach-Object -ThrottleLimit (Get-PSFConfigValue -FullName 'AzOps.Core.ThrottleLimit') -Parallel { @@ -225,7 +237,10 @@ else { $query = "resourcecontainers | where type == 'microsoft.resources/subscriptions/resourcegroups' | where managedBy == '' | order by ['id'] asc" if ($SubscriptionsToIncludeResourceGroups -ne '*') { - $newSubscriptionsToIncludeResourceGroups = $subscriptions | Where-Object { $_.Id -in $SubscriptionsToIncludeResourceGroups } + $newSubscriptionsToIncludeResourceGroups = $subscriptions | Where-Object { + $_.Id -in $SubscriptionsToIncludeResourceGroups -and + $_.Id -notin $SkipSubscription + } if ($newSubscriptionsToIncludeResourceGroups) { $resourceGroups = Search-AzOpsAzGraph -Subscription $newSubscriptionsToIncludeResourceGroups -Query $query -ErrorAction Stop } diff --git a/src/internal/functions/Search-AzOpsAzGraph.ps1 b/src/internal/functions/Search-AzOpsAzGraph.ps1 index 9f13fd86..a1ecac50 100644 --- a/src/internal/functions/Search-AzOpsAzGraph.ps1 +++ b/src/internal/functions/Search-AzOpsAzGraph.ps1 @@ -95,12 +95,10 @@ $batchSize = 1000 # Group subscriptions into batches to conform with graph limits $subscriptionBatch = $Subscription | Group-Object -Property { [math]::Floor($counter++ / $batchSize) } - foreach ($group in $subscriptionBatch) { $subscriptionIds = ($group.Group).Id -join ', ' $subscriptionCount = $group.Group.Count Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.SubscriptionBatch' -LogStringValues $subscriptionCount, $subscriptionIds - try { $batchProcessing = $null do { @@ -112,8 +110,7 @@ catch { # Batch failed - try each subscription individually to identify the problematic scope Write-AzOpsMessage -LogLevel Warning -LogString 'Search-AzOpsAzGraph.Processing.SubscriptionBatch.Failed' -LogStringValues $subscriptionIds, $_.Exception.Message - Write-AzOpsMessage -LogLevel Important -LogString 'Search-AzOpsAzGraph.Processing.SubscriptionBatch.RetryIndividually' -LogStringValues $subscriptionCount - + Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.SubscriptionBatch.RetryIndividually' -LogStringValues $subscriptionCount foreach ($sub in $group.Group) { try { Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.Subscription' -LogStringValues $sub.Name, $sub.Id @@ -126,25 +123,64 @@ } catch { Write-AzOpsMessage -LogLevel Error -LogString 'Search-AzOpsAzGraph.Processing.Subscription.Failed' -LogStringValues $Query, $sub.Name, $sub.Id, $_.Exception.Message - try { Write-AzOpsMessage -LogLevel Debug -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RetryWithRestApi' -LogStringValues $sub.Id $resourceGraphApiVersion = (($script:AzOpsResourceProvider | Where-Object {$_.ProviderNamespace -eq 'Microsoft.ResourceGraph'}).ResourceTypes | Where-Object {$_.ResourceTypeName -eq 'queries'}).ApiVersions | Select-Object -First 1 - # Initialize request body for first call $requestBody = @{ subscriptions = @($sub.Id) query = $Query } | ConvertTo-Json -Depth 10 - $restApiResponse = $null do { $response = Invoke-AzRestMethod -Method POST -Path "/providers/Microsoft.ResourceGraph/resources?api-version=$resourceGraphApiVersion" -Payload $requestBody -ErrorAction Stop - if ($response.StatusCode -eq 200) { - $restApiResponse = $response.Content | ConvertFrom-Json -Depth 100 -ErrorAction Stop + try { + $restApiResponse = $response.Content | ConvertFrom-Json -Depth 100 -ErrorAction Stop + } + catch { + # Fallback to hashtable for empty string property names + try { + #Write-AzOpsMessage -LogLevel Debug -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RetryAsHashtable' -LogStringValues $sub.Id + $restApiResponse = $response.Content | ConvertFrom-Json -Depth 100 -AsHashtable -ErrorAction Stop + # Identify which resource caused the need for -AsHashtable + if ($restApiResponse['data']) { + # Store skipToken before processing + $originalSkipToken = $restApiResponse['$skipToken'] + $cleanData = [System.Collections.Generic.List[object]]::new() + foreach ($resource in $restApiResponse['data']) { + # Check if resource contains empty string keys by converting to JSON and checking + $resourceJson = $resource | ConvertTo-Json -Depth 100 -Compress + if ($resourceJson -match '"":\s*[^,}]') { + $id = $resource['id'] + Write-AzOpsMessage -LogLevel Warning -LogString 'Search-AzOpsAzGraph.Processing.Subscription.EmptyStringKeyDetected' -LogStringValues $id + # Skip this resource - don't add it to cleaned data + continue + } + # Add valid resources to the cleaned list + $cleanData.Add($resource) + } + # Convert hashtable back to PSCustomObject structure + $restApiResponse = [PSCustomObject]@{ + data = $cleanData | ForEach-Object { $_ | ConvertTo-Json -Depth 100 | ConvertFrom-Json -Depth 100 } + totalRecords = $restApiResponse['totalRecords'] + count = $cleanData.Count + facets = $restApiResponse['facets'] + } + + # Restore skipToken if it existed + if ($originalSkipToken) { + $restApiResponse | Add-Member -MemberType NoteProperty -Name '$skipToken' -Value $originalSkipToken + } + } + } + catch { + Write-AzOpsMessage -LogLevel Error -LogString 'Search-AzOpsAzGraph.Processing.Subscription.JsonParseFailed' -LogStringValues $Query, $sub.Id, $_.Exception.Message + # Skip to next subscription + } + } if ($restApiResponse.data) { - Write-AzOpsMessage -LogLevel Important -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RestApiSuccess' -LogStringValues $sub.Id, $restApiResponse.data.Count + Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RestApiSuccess' -LogStringValues $sub.Id, $restApiResponse.data.Count $results.AddRange($restApiResponse.data) } @@ -162,7 +198,6 @@ else { # Log the raw error response for analysis Write-AzOpsMessage -LogLevel Error -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RestApiFailed' -LogStringValues $sub.Id, $response.StatusCode, $response.Content - # Attempt to parse error details try { $errorContent = $response.Content | ConvertFrom-Json -ErrorAction Stop @@ -188,7 +223,7 @@ } } } - + if ($results) { $providerLookup = @{} foreach ($ResourceProvider in $script:AzOpsResourceProvider) { diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index dbcbcca1..7f5ae18e 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -350,7 +350,9 @@ 'Search-AzOpsAzGraph.Processing.ManagementGroup' = 'AzGraph processing query at managementGroup: [{0}]' # $ManagementGroupName 'Search-AzOpsAzGraph.Processing.ManagementGroup.Failed' = 'Failed AzGraph processing query: [{0}] at managementGroup: [{1}] with: {2}' # $Query, $ManagementGroupName, $_.Exception.Message 'Search-AzOpsAzGraph.Processing.Subscription' = 'AzGraph processing query at subscription: [{0}({1})]' # $sub.Name, $sub.Id + 'Search-AzOpsAzGraph.Processing.Subscription.EmptyStringKeyDetected' = 'Failed AzGraph processing of resource: [{0}] with missing key name, excluding resource from processing' # $id 'Search-AzOpsAzGraph.Processing.Subscription.Failed' = 'Failed AzGraph processing query: [{0}] at subscription: [{1}({2})] with: {3}' # $Query, $sub.Name, $sub.Id, $_.Exception.Message + 'Search-AzOpsAzGraph.Processing.Subscription.JsonParseFailed' = 'Failed AzGraph processing query: [{0}] at subscription: [{1}] with: {2}' # $Query, $sub.Id, $_.Exception.Message 'Search-AzOpsAzGraph.Processing.Subscription.RetryWithRestApi' = 'Retrying Microsoft.ResourceGraph query at subscription [{0}] using REST API for additional diagnostics' # $sub.Id 'Search-AzOpsAzGraph.Processing.Subscription.RestApiSuccess' = 'Microsoft.ResourceGraph REST API query succeeded for subscription [{0}] with [{1}] results' # $sub.Id, $responseContent.data.Count 'Search-AzOpsAzGraph.Processing.Subscription.RestApiFailed' = 'Microsoft.ResourceGraph REST API query failed for subscription [{0}] with status [{1}]: {2}' # $sub.Id, $response.StatusCode, $response.Content From 19bedcee565bae2999331067ebff7f211b8ebfd1 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 7 Oct 2025 14:46:28 +0000 Subject: [PATCH 05/13] Update --- .../functions/Get-AzOpsNestedSubscription.ps1 | 17 ++++++++++++----- .../functions/Get-AzOpsResourceDefinition.ps1 | 4 ---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/internal/functions/Get-AzOpsNestedSubscription.ps1 b/src/internal/functions/Get-AzOpsNestedSubscription.ps1 index 76b31133..8eeff868 100644 --- a/src/internal/functions/Get-AzOpsNestedSubscription.ps1 +++ b/src/internal/functions/Get-AzOpsNestedSubscription.ps1 @@ -4,6 +4,8 @@ Create a list of subscriptionId's nested at ManagementGroup Scope .PARAMETER Scope ManagementGroup Name + .PARAMETER SkipSubscription + Filter which Subscription IDs should be excluded from pull. .EXAMPLE > Get-AzOpsNestedSubscription -Scope 5663f39e-feb1-4303-a1f9-cf20b702de61 Discover subscriptions at Management Group scope and below @@ -13,7 +15,10 @@ param ( [Parameter(Mandatory = $false)] [string] - $Scope + $Scope, + + [string[]] + $SkipSubscription = (Get-PSFConfigValue -FullName 'AzOps.Core.SkipSubscription') ) process { @@ -22,10 +27,12 @@ $subscriptionIds = @() foreach ($child in $children) { if (($child.Type -eq '/subscriptions') -and ($script:AzOpsSubscriptions.id -contains $child.Id)) { - $subscriptionIds += [PSCustomObject] @{ - Name = $child.DisplayName - Id = $child.Name - Type = $child.Type + if ($child.Name -notin $SkipSubscription) { + $subscriptionIds += [PSCustomObject] @{ + Name = $child.DisplayName + Id = $child.Name + Type = $child.Type + } } } else { diff --git a/src/internal/functions/Get-AzOpsResourceDefinition.ps1 b/src/internal/functions/Get-AzOpsResourceDefinition.ps1 index 0b483273..519560bd 100644 --- a/src/internal/functions/Get-AzOpsResourceDefinition.ps1 +++ b/src/internal/functions/Get-AzOpsResourceDefinition.ps1 @@ -146,10 +146,6 @@ $query = "resourcecontainers | where type == 'microsoft.management/managementgroups' | order by ['id'] asc" $managementgroups = Search-AzOpsAzGraph -ManagementGroupName $scopeObject.Name -Query $query -ErrorAction Stop | Where-Object { $_.id -in $script:AzOpsAzManagementGroup.Id } $subscriptions = Get-AzOpsNestedSubscription -Scope $scopeObject.Name - # Filter out skipped subscriptions - if ($SkipSubscription) { - $subscriptions = $subscriptions | Where-Object { $_.Id -notin $SkipSubscription } - } if ($managementgroups) { # Process managementGroup scope in parallel $managementgroups | Foreach-Object -ThrottleLimit (Get-PSFConfigValue -FullName 'AzOps.Core.ThrottleLimit') -Parallel { From eb6f0fe970fbed2f32c9500841e8badc086cfda6 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 8 Oct 2025 08:44:51 +0000 Subject: [PATCH 06/13] Update --- .../functions/Get-AzOpsNestedSubscription.ps1 | 12 ++++----- .../functions/Search-AzOpsAzGraph.ps1 | 27 ++++++++++++------- src/localized/en-us/Strings.psd1 | 1 + 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/internal/functions/Get-AzOpsNestedSubscription.ps1 b/src/internal/functions/Get-AzOpsNestedSubscription.ps1 index 8eeff868..c8685f0a 100644 --- a/src/internal/functions/Get-AzOpsNestedSubscription.ps1 +++ b/src/internal/functions/Get-AzOpsNestedSubscription.ps1 @@ -26,13 +26,11 @@ if ($children) { $subscriptionIds = @() foreach ($child in $children) { - if (($child.Type -eq '/subscriptions') -and ($script:AzOpsSubscriptions.id -contains $child.Id)) { - if ($child.Name -notin $SkipSubscription) { - $subscriptionIds += [PSCustomObject] @{ - Name = $child.DisplayName - Id = $child.Name - Type = $child.Type - } + if (($child.Type -eq '/subscriptions') -and ($script:AzOpsSubscriptions.id -contains $child.Id) -and ($child.Name -notin $SkipSubscription)) { + $subscriptionIds += [PSCustomObject] @{ + Name = $child.DisplayName + Id = $child.Name + Type = $child.Type } } else { diff --git a/src/internal/functions/Search-AzOpsAzGraph.ps1 b/src/internal/functions/Search-AzOpsAzGraph.ps1 index a1ecac50..a35d9a44 100644 --- a/src/internal/functions/Search-AzOpsAzGraph.ps1 +++ b/src/internal/functions/Search-AzOpsAzGraph.ps1 @@ -66,7 +66,8 @@ try { do { $tenantProcessing = Search-AzGraph -UseTenantScope -Query $Query -AllowPartialScope -SkipToken $tenantProcessing.SkipToken -ErrorAction Stop - if ($tenantProcessing) { $results.AddRange($tenantProcessing) } + if ($tenantProcessing -and $tenantProcessing -is [array]) { $results.AddRange($tenantProcessing) } + elseif ($tenantProcessing) { $results.Add($tenantProcessing) } } while ($tenantProcessing.SkipToken) } @@ -80,7 +81,8 @@ try { do { $mgProcessing = Search-AzGraph -ManagementGroup $ManagementGroupName -Query $Query -AllowPartialScope -SkipToken $mgProcessing.SkipToken -ErrorAction Stop - if ($mgProcessing) { $results.AddRange($mgProcessing) } + if ($mgProcessing -and $mgProcessing -is [array]) { $results.AddRange($mgProcessing) } + elseif ($mgProcessing) { $results.Add($mgProcessing) } } while ($mgProcessing.SkipToken) } @@ -103,7 +105,8 @@ $batchProcessing = $null do { $batchProcessing = Search-AzGraph -Subscription ($group.Group).Id -Query $Query -SkipToken $batchProcessing.SkipToken -ErrorAction Stop - if ($batchProcessing) { $results.AddRange($batchProcessing) } + if ($batchProcessing -and $batchProcessing -is [array]) { $results.AddRange($batchProcessing) } + elseif ($batchProcessing) { $results.Add($batchProcessing) } } while ($batchProcessing.SkipToken) } @@ -117,7 +120,8 @@ $subProcessing = $null do { $subProcessing = Search-AzGraph -Subscription $sub.Id -Query $Query -SkipToken $subProcessing.SkipToken -ErrorAction Stop - if ($subProcessing) { $results.AddRange($subProcessing) } + if ($subProcessing -and $subProcessing -is [array]) { $results.AddRange($subProcessing) } + elseif ($subProcessing) { $results.Add($subProcessing) } } while ($subProcessing.SkipToken) } @@ -140,8 +144,12 @@ catch { # Fallback to hashtable for empty string property names try { - #Write-AzOpsMessage -LogLevel Debug -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RetryAsHashtable' -LogStringValues $sub.Id $restApiResponse = $response.Content | ConvertFrom-Json -Depth 100 -AsHashtable -ErrorAction Stop + # Validate response structure + if (-not $restApiResponse.ContainsKey('data')) { + Write-AzOpsMessage -LogLevel Warning -LogString 'Search-AzOpsAzGraph.Processing.Subscription.InvalidRestApiResponse' -LogStringValues $requestBody, $_.Exception.Message + break + } # Identify which resource caused the need for -AsHashtable if ($restApiResponse['data']) { # Store skipToken before processing @@ -178,12 +186,11 @@ # Skip to next subscription } } - if ($restApiResponse.data) { Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RestApiSuccess' -LogStringValues $sub.Id, $restApiResponse.data.Count - $results.AddRange($restApiResponse.data) + if ($restApiResponse.data -is [array]) { $results.AddRange($restApiResponse.data) } + else { $results.Add($restApiResponse.data) } } - # Prepare next page request if skipToken exists if ($restApiResponse.'$skipToken') { $requestBody = @{ @@ -229,7 +236,7 @@ foreach ($ResourceProvider in $script:AzOpsResourceProvider) { foreach ($ResourceTypeName in $ResourceProvider.ResourceTypes.ResourceTypeName) { # Use lowercase key for case-insensitive matching - $key = "$($ResourceProvider.ProviderNamespace)/$ResourceTypeName".ToLower() + $key = "$($ResourceProvider.ProviderNamespace)/$ResourceTypeName".ToLower([System.Globalization.CultureInfo]::InvariantCulture) $providerLookup[$key] = @{ Namespace = $ResourceProvider.ProviderNamespace TypeName = $ResourceTypeName @@ -240,7 +247,7 @@ $resultsType = [System.Collections.Generic.List[object]]::new() foreach ($result in $results) { # Process each graph result and normalize ProviderNamespace casing using hashtable lookup - $resultTypeKey = $result.type.ToLower() + $resultTypeKey = $result.type.ToLower([System.Globalization.CultureInfo]::InvariantCulture) if ($providerLookup.ContainsKey($resultTypeKey)) { # Reconstruct the type with correct casing from the lookup $result.type = "$($providerLookup[$resultTypeKey].Namespace)/$($providerLookup[$resultTypeKey].TypeName)" diff --git a/src/localized/en-us/Strings.psd1 b/src/localized/en-us/Strings.psd1 index 7f5ae18e..02125441 100644 --- a/src/localized/en-us/Strings.psd1 +++ b/src/localized/en-us/Strings.psd1 @@ -351,6 +351,7 @@ 'Search-AzOpsAzGraph.Processing.ManagementGroup.Failed' = 'Failed AzGraph processing query: [{0}] at managementGroup: [{1}] with: {2}' # $Query, $ManagementGroupName, $_.Exception.Message 'Search-AzOpsAzGraph.Processing.Subscription' = 'AzGraph processing query at subscription: [{0}({1})]' # $sub.Name, $sub.Id 'Search-AzOpsAzGraph.Processing.Subscription.EmptyStringKeyDetected' = 'Failed AzGraph processing of resource: [{0}] with missing key name, excluding resource from processing' # $id + 'Search-AzOpsAzGraph.Processing.Subscription.InvalidRestApiResponse' = 'Failed AzGraph processing of request [{0}] with invalid REST API response (missing data): [{2}], excluding resource from processing' # $requestBody, $_.Exception.Message 'Search-AzOpsAzGraph.Processing.Subscription.Failed' = 'Failed AzGraph processing query: [{0}] at subscription: [{1}({2})] with: {3}' # $Query, $sub.Name, $sub.Id, $_.Exception.Message 'Search-AzOpsAzGraph.Processing.Subscription.JsonParseFailed' = 'Failed AzGraph processing query: [{0}] at subscription: [{1}] with: {2}' # $Query, $sub.Id, $_.Exception.Message 'Search-AzOpsAzGraph.Processing.Subscription.RetryWithRestApi' = 'Retrying Microsoft.ResourceGraph query at subscription [{0}] using REST API for additional diagnostics' # $sub.Id From 9732383783ec0d420e72369907b2ccd06b0c3c0d Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 8 Oct 2025 12:40:42 +0000 Subject: [PATCH 07/13] Update --- src/internal/functions/Search-AzOpsAzGraph.ps1 | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/internal/functions/Search-AzOpsAzGraph.ps1 b/src/internal/functions/Search-AzOpsAzGraph.ps1 index a35d9a44..d0e3a218 100644 --- a/src/internal/functions/Search-AzOpsAzGraph.ps1 +++ b/src/internal/functions/Search-AzOpsAzGraph.ps1 @@ -66,8 +66,7 @@ try { do { $tenantProcessing = Search-AzGraph -UseTenantScope -Query $Query -AllowPartialScope -SkipToken $tenantProcessing.SkipToken -ErrorAction Stop - if ($tenantProcessing -and $tenantProcessing -is [array]) { $results.AddRange($tenantProcessing) } - elseif ($tenantProcessing) { $results.Add($tenantProcessing) } + if ($tenantProcessing) { $results.AddRange($tenantProcessing) } } while ($tenantProcessing.SkipToken) } @@ -81,8 +80,7 @@ try { do { $mgProcessing = Search-AzGraph -ManagementGroup $ManagementGroupName -Query $Query -AllowPartialScope -SkipToken $mgProcessing.SkipToken -ErrorAction Stop - if ($mgProcessing -and $mgProcessing -is [array]) { $results.AddRange($mgProcessing) } - elseif ($mgProcessing) { $results.Add($mgProcessing) } + if ($mgProcessing) { $results.AddRange($mgProcessing) } } while ($mgProcessing.SkipToken) } @@ -105,8 +103,7 @@ $batchProcessing = $null do { $batchProcessing = Search-AzGraph -Subscription ($group.Group).Id -Query $Query -SkipToken $batchProcessing.SkipToken -ErrorAction Stop - if ($batchProcessing -and $batchProcessing -is [array]) { $results.AddRange($batchProcessing) } - elseif ($batchProcessing) { $results.Add($batchProcessing) } + if ($batchProcessing) { $results.AddRange($batchProcessing) } } while ($batchProcessing.SkipToken) } @@ -120,8 +117,7 @@ $subProcessing = $null do { $subProcessing = Search-AzGraph -Subscription $sub.Id -Query $Query -SkipToken $subProcessing.SkipToken -ErrorAction Stop - if ($subProcessing -and $subProcessing -is [array]) { $results.AddRange($subProcessing) } - elseif ($subProcessing) { $results.Add($subProcessing) } + if ($subProcessing) { $results.AddRange($subProcessing) } } while ($subProcessing.SkipToken) } @@ -188,8 +184,7 @@ } if ($restApiResponse.data) { Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.Subscription.RestApiSuccess' -LogStringValues $sub.Id, $restApiResponse.data.Count - if ($restApiResponse.data -is [array]) { $results.AddRange($restApiResponse.data) } - else { $results.Add($restApiResponse.data) } + $results.AddRange($restApiResponse.data) } # Prepare next page request if skipToken exists if ($restApiResponse.'$skipToken') { From 2e5dd1d124ad7831160d2358ec34b00018d118f5 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Wed, 8 Oct 2025 13:08:36 +0000 Subject: [PATCH 08/13] Update --- src/internal/functions/Search-AzOpsAzGraph.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/internal/functions/Search-AzOpsAzGraph.ps1 b/src/internal/functions/Search-AzOpsAzGraph.ps1 index d0e3a218..a97e0fd8 100644 --- a/src/internal/functions/Search-AzOpsAzGraph.ps1 +++ b/src/internal/functions/Search-AzOpsAzGraph.ps1 @@ -241,6 +241,10 @@ $resultsType = [System.Collections.Generic.List[object]]::new() foreach ($result in $results) { + # Add null check for result.type property + if (-not $result.type) { + continue + } # Process each graph result and normalize ProviderNamespace casing using hashtable lookup $resultTypeKey = $result.type.ToLower([System.Globalization.CultureInfo]::InvariantCulture) if ($providerLookup.ContainsKey($resultTypeKey)) { From a958ce2a19503b0a4735f74dcf9557178ec7c22e Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 9 Oct 2025 08:44:40 +0000 Subject: [PATCH 09/13] Update --- src/internal/functions/Search-AzOpsAzGraph.ps1 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/internal/functions/Search-AzOpsAzGraph.ps1 b/src/internal/functions/Search-AzOpsAzGraph.ps1 index a97e0fd8..d505bdee 100644 --- a/src/internal/functions/Search-AzOpsAzGraph.ps1 +++ b/src/internal/functions/Search-AzOpsAzGraph.ps1 @@ -60,7 +60,7 @@ process { Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing' -LogStringValues $Query $results = [System.Collections.Generic.List[object]]::new() - + if ($UseTenantScope) { Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.UseTenantScope' try { @@ -74,7 +74,7 @@ Write-AzOpsMessage -LogLevel Error -LogString 'Search-AzOpsAzGraph.Processing.UseTenantScope.Failed' -LogStringValues $Query, $_.Exception.Message } } - + if ($ManagementGroupName) { Write-AzOpsMessage -LogLevel Verbose -LogString 'Search-AzOpsAzGraph.Processing.ManagementGroup' -LogStringValues $ManagementGroupName try { @@ -88,13 +88,13 @@ Write-AzOpsMessage -LogLevel Error -LogString 'Search-AzOpsAzGraph.Processing.ManagementGroup.Failed' -LogStringValues $Query, $ManagementGroupName, $_.Exception.Message } } - + if ($Subscription) { # Create a counter, set the batch size, and prepare a variable for the results - $counter = 0 + $counter = [PSCustomObject] @{ Value = 0 } $batchSize = 1000 # Group subscriptions into batches to conform with graph limits - $subscriptionBatch = $Subscription | Group-Object -Property { [math]::Floor($counter++ / $batchSize) } + $subscriptionBatch = $Subscription | Group-Object -Property { [math]::Floor($counter.Value++ / $batchSize) } foreach ($group in $subscriptionBatch) { $subscriptionIds = ($group.Group).Id -join ', ' $subscriptionCount = $group.Group.Count @@ -238,7 +238,7 @@ } } } - + $resultsType = [System.Collections.Generic.List[object]]::new() foreach ($result in $results) { # Add null check for result.type property From f492b4fff7906f8eb55ea0a2da363c8d186dbbeb Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 10 Oct 2025 07:55:52 +0000 Subject: [PATCH 10/13] Add SkipSubscription Test --- src/tests/integration/Repository.Tests.ps1 | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 6fe3e247..79ae41bc 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -22,6 +22,7 @@ Describe "Repository" { $script:tenantId = $env:ARM_TENANT_ID $script:subscriptionId = $env:ARM_SUBSCRIPTION_ID $otherSubscription = Get-AzSubscription | Where-Object { $_.Id -ne $script:subscriptionId } | Sort-Object Name -Descending | Select-Object Id -First 2 + $skipSubscription = Get-AzSubscription | Where-Object { $_.Id -ne $script:subscriptionId -and $_.Id -notin $otherSubscription.Id } | Sort-Object Name -Descending | Select-Object Id -First 1 # Validate that the runtime variables are set as they are used to authenticate the Azure session. @@ -201,6 +202,7 @@ Describe "Repository" { Set-PSFConfig -FullName AzOps.Core.State -Value $generatedRoot Set-PSFConfig -FullName AzOps.Core.SkipLock -Value $false Set-PSFConfig -FullName AzOps.Core.SkipChildResource -Value $false + Set-PSFConfig -FullName AzOps.Core.SkipSubscription -Value @($skipSubscription.Id) Set-PSFConfig -FullName AzOps.Core.DefaultDeploymentRegion -Value "northeurope" $deploymentLocationId = (Get-FileHash -Algorithm SHA256 -InputStream ([IO.MemoryStream]::new([byte[]][char[]](Get-PSFConfigValue -FullName 'AzOps.Core.DefaultDeploymentRegion')))).Hash.Substring(0, 4) @@ -1363,6 +1365,7 @@ Describe "Repository" { Get-AzPolicyAssignment -Id $script:policyAssignmentsDeletion.Id -ErrorAction SilentlyContinue | Should -Be $Null } #endregion + #region AzOps Managed DeploymentStacks It "Deploy and Delete AzOps Managed DeploymentStacks at Resource Group Scope" { Set-PSFConfig -FullName AzOps.Core.CustomTemplateResourceDeletion -Value $true @@ -1457,6 +1460,27 @@ Describe "Repository" { } } #endregion + + #region SkipSubscription Filter + It "Should have skipped subscription file present but no child resources" { + # Check that the skipped subscription file exists + $skippedSubscriptionFiles = $filePaths | Where-Object Name -eq "microsoft.subscription_subscriptions-$(($skipSubscription.Id).toLower()).json" + $skippedSubscriptionFiles.Count | Should -Be 1 + + # Get the directory of the skipped subscription + $skippedSubscriptionDirectory = ($skippedSubscriptionFiles).Directory + + # Check that the directory exists + Test-Path -Path $skippedSubscriptionDirectory | Should -BeTrue + + # Get all items in the subscription directory + $allItems = Get-ChildItem -Path $skippedSubscriptionDirectory -Recurse + + # Should only have the subscription file itself, nothing else + $allItems.Count | Should -Be 1 + $allItems[0].Name | Should -Be "microsoft.subscription_subscriptions-$(($skipSubscription.Id).toLower()).json" + } + #endregion } AfterAll { From c7ddf7b0d6b02b48ba6113d2142cea6dd9a62b3a Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Fri, 17 Oct 2025 12:27:56 +0000 Subject: [PATCH 11/13] Update --- src/AzOps.psd1 | 6 +- src/tests/integration/Repository.Tests.ps1 | 29 +++++ src/tests/templates/azuredeploy.jsonc | 133 +++++++++++++++++++++ 3 files changed, 165 insertions(+), 3 deletions(-) diff --git a/src/AzOps.psd1 b/src/AzOps.psd1 index 0a7b08b0..ad171ed3 100644 --- a/src/AzOps.psd1 +++ b/src/AzOps.psd1 @@ -3,7 +3,7 @@ # # Generated by: Customer Architecture Team (CAT) # -# Generated on: 09/16/2025 +# Generated on: 10/17/2025 # @{ @@ -51,11 +51,11 @@ PowerShellVersion = '7.2' # ProcessorArchitecture = '' # Modules that must be imported into the global environment prior to importing this module -RequiredModules = @(@{ModuleName = 'PSFramework'; RequiredVersion = '1.13.406'; }, +RequiredModules = @(@{ModuleName = 'PSFramework'; RequiredVersion = '1.13.414'; }, @{ModuleName = 'Az.Accounts'; RequiredVersion = '5.3.0'; }, @{ModuleName = 'Az.Billing'; RequiredVersion = '2.2.0'; }, @{ModuleName = 'Az.ResourceGraph'; RequiredVersion = '1.2.1'; }, - @{ModuleName = 'Az.Resources'; RequiredVersion = '8.1.0'; }) + @{ModuleName = 'Az.Resources'; RequiredVersion = '8.1.1'; }) # Assemblies that must be loaded prior to importing this module # RequiredAssemblies = @() diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 79ae41bc..1f76be1a 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -1481,6 +1481,35 @@ Describe "Repository" { $allItems[0].Name | Should -Be "microsoft.subscription_subscriptions-$(($skipSubscription.Id).toLower()).json" } #endregion + + #region Scope - Broken Logic App Resource Group (./root/tenant root group/test/platform/management/subscription-0/BrokenLogicApp-azopsrg) + It "Broken Logic App Resource Group directory should exist" { + $script:resourceGroupBrokenLogicAppPath = ($filePaths | Where-Object Name -eq "microsoft.resources_resourcegroups-brokenlogicapp-azopsrg.json") + $script:resourceGroupBrokenLogicAppDirectory = ($script:resourceGroupBrokenLogicAppPath).Directory + $script:resourceGroupBrokenLogicAppFile = ($script:resourceGroupBrokenLogicAppPath).FullName + Test-Path -Path $script:resourceGroupBrokenLogicAppDirectory | Should -BeTrue + } + + It "Broken Logic App Resource Group file should exist" { + Test-Path -Path $script:resourceGroupBrokenLogicAppFile | Should -BeTrue + } + + It "Working Logic App should be present in pulled resources" { + $workingLogicAppPath = Get-ChildItem -Path $script:resourceGroupBrokenLogicAppDirectory -Recurse -Filter "*my-xworking-logic-app*.json" + $workingLogicAppPath | Should -Not -BeNullOrEmpty + $workingLogicAppPath.Count | Should -Be 1 + } + + It "Broken Logic App should NOT be present in pulled resources (excluded due to retry failure)" { + $brokenLogicAppPath = Get-ChildItem -Path $script:resourceGroupBrokenLogicAppDirectory -Recurse -Filter "*my-xbroken-logic-app*.json" + $brokenLogicAppPath | Should -BeNullOrEmpty + } + + It "Should have exactly 2 files in BrokenLogicApp-azopsrg directory (Resource Group + Working Logic App)" { + $allFiles = Get-ChildItem -Path $script:resourceGroupBrokenLogicAppDirectory -Recurse -File + $allFiles.Count | Should -Be 2 + } + #endregion } AfterAll { diff --git a/src/tests/templates/azuredeploy.jsonc b/src/tests/templates/azuredeploy.jsonc index b004bd5d..af6c9088 100644 --- a/src/tests/templates/azuredeploy.jsonc +++ b/src/tests/templates/azuredeploy.jsonc @@ -575,6 +575,12 @@ "name": "DeploymentStacksDeploy-azopsrg", "location": "northeurope" }, + { + "type": "Microsoft.Resources/resourceGroups", + "apiVersion": "2024-11-01", + "name": "BrokenLogicApp-azopsrg", + "location": "northeurope" + }, { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", @@ -861,6 +867,133 @@ } } } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2024-11-01", + "name": "BrokenLogicApp", + "resourceGroup": "BrokenLogicApp-azopsrg", + "dependsOn": [ + "BrokenLogicApp-azopsrg" + ], + "properties": { + "mode": "Incremental", + "expressionEvaluationOptions": { + "scope": "inner" + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Logic/workflows", + "apiVersion": "2019-05-01", + "name": "my-xbroken-logic-app", + "location": "swedencentral", + "properties": { + "state": "Enabled", + "definition": { + "$schema": "https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "$connections": { + "defaultValue": {}, + "type": "Object" + } + }, + "triggers": { + "Recurrence": { + "recurrence": { + "interval": 3, + "frequency": "Month", + "timeZone": "Dateline Standard Time" + }, + "evaluatedRecurrence": { + "interval": 3, + "frequency": "Month", + "timeZone": "Dateline Standard Time" + }, + "type": "Recurrence" + } + }, + "actions": { + "Select": { + "runAfter": {}, + "type": "Select", + "inputs": { + "from": "@guid()", + "select": { + "": "valueofkey" + } + } + } + }, + "outputs": {} + }, + "parameters": { + "$connections": { + "value": {} + } + } + } + }, + { + "type": "Microsoft.Logic/workflows", + "apiVersion": "2019-05-01", + "name": "my-xworking-logic-app", + "location": "swedencentral", + "properties": { + "state": "Enabled", + "definition": { + "$schema": "https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "$connections": { + "defaultValue": {}, + "type": "Object" + } + }, + "triggers": { + "Recurrence": { + "recurrence": { + "interval": 3, + "frequency": "Month", + "timeZone": "Dateline Standard Time" + }, + "evaluatedRecurrence": { + "interval": 3, + "frequency": "Month", + "timeZone": "Dateline Standard Time" + }, + "type": "Recurrence" + } + }, + "actions": { + "Select": { + "runAfter": {}, + "type": "Select", + "inputs": { + "from": "@guid()", + "select": { + "keyname": "valueofkey" + } + } + } + }, + "outputs": {} + }, + "parameters": { + "$connections": { + "value": {} + } + } + } + } + ], + "outputs": { + } + } + } } ] } From ab9266ef32282c6663d071e2f5e4bb2aacdaa8cf Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Sat, 18 Oct 2025 12:28:12 +0000 Subject: [PATCH 12/13] Update --- src/tests/integration/Repository.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 1f76be1a..e875e3bd 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -1305,7 +1305,7 @@ Describe "Repository" { $maxParallel = ($parallelTimes | Measure-Object -Maximum).Maximum $minParallel = ($parallelTimes | Measure-Object -Minimum).Minimum $diffParallel = New-TimeSpan -Start $minParallel -End $maxParallel - $diffParallel.TotalSeconds | Should -BeLessThan 25 + $diffParallel.TotalSeconds | Should -BeLessThan 35 $serialTime = $serialDeployment.Timestamp $diffSerial = New-TimeSpan -Start $maxParallel -End $serialTime $diffSerial.TotalSeconds | Should -BeGreaterThan 15 From 5896bc6198365f192ff1830b3f5682d190c2b5d8 Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Thu, 27 Nov 2025 09:27:36 +0000 Subject: [PATCH 13/13] Update --- src/AzOps.psd1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/AzOps.psd1 b/src/AzOps.psd1 index ad171ed3..326a47dc 100644 --- a/src/AzOps.psd1 +++ b/src/AzOps.psd1 @@ -3,7 +3,7 @@ # # Generated by: Customer Architecture Team (CAT) # -# Generated on: 10/17/2025 +# Generated on: 11/27/2025 # @{ @@ -51,11 +51,11 @@ PowerShellVersion = '7.2' # ProcessorArchitecture = '' # Modules that must be imported into the global environment prior to importing this module -RequiredModules = @(@{ModuleName = 'PSFramework'; RequiredVersion = '1.13.414'; }, - @{ModuleName = 'Az.Accounts'; RequiredVersion = '5.3.0'; }, +RequiredModules = @(@{ModuleName = 'PSFramework'; RequiredVersion = '1.13.419'; }, + @{ModuleName = 'Az.Accounts'; RequiredVersion = '5.3.1'; }, @{ModuleName = 'Az.Billing'; RequiredVersion = '2.2.0'; }, @{ModuleName = 'Az.ResourceGraph'; RequiredVersion = '1.2.1'; }, - @{ModuleName = 'Az.Resources'; RequiredVersion = '8.1.1'; }) + @{ModuleName = 'Az.Resources'; RequiredVersion = '9.0.0'; }) # Assemblies that must be loaded prior to importing this module # RequiredAssemblies = @()