From 2174f538cb620b2eebfa7982d3b755f4bac7cad9 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 25 Nov 2024 16:26:17 -0500 Subject: [PATCH 001/119] Initial --- scenarios/AksOpenAiTerraform/README.md | 18 + scenarios/AksOpenAiTerraform/scripts/.env | 2 + .../scripts/00-variables.sh | 71 ++ .../scripts/01-build-docker-image.sh | 12 + .../scripts/02-run-docker-container.sh | 21 + .../scripts/03-push-docker-image.sh | 16 + .../04-create-nginx-ingress-controller.sh | 39 + .../scripts/05-install-cert-manager.sh | 34 + .../scripts/06-create-cluster-issuer.sh | 19 + .../07-create-workload-managed-identity.sh | 104 +++ .../scripts/08-create-service-account.sh | 103 +++ .../scripts/09-deploy-app.sh | 46 ++ .../scripts/10-create-ingress.sh | 12 + .../scripts/11-configure-dns.sh | 79 ++ .../AksOpenAiTerraform/scripts/Dockerfile | 94 +++ scenarios/AksOpenAiTerraform/scripts/app.py | 347 ++++++++ .../scripts/cluster-issuer.yml | 18 + .../AksOpenAiTerraform/scripts/configMap.yml | 14 + .../AksOpenAiTerraform/scripts/deployment.yml | 123 +++ .../scripts/images/magic8ball.png | Bin 0 -> 37452 bytes .../scripts/images/robot.png | Bin 0 -> 1686 bytes .../AksOpenAiTerraform/scripts/ingress.yml | 30 + .../scripts/requirements.txt | 145 ++++ .../AksOpenAiTerraform/scripts/service.yml | 13 + .../install-nginx-via-helm-and-create-sa.sh | 218 +++++ .../AksOpenAiTerraform/terraform/main.tf | 454 +++++++++++ .../terraform/modules/aks/main.tf | 180 +++++ .../terraform/modules/aks/outputs.tf | 40 + .../terraform/modules/aks/variables.tf | 316 ++++++++ .../terraform/modules/bastion_host/main.tf | 99 +++ .../terraform/modules/bastion_host/output.tf | 23 + .../modules/bastion_host/variables.tf | 35 + .../modules/container_registry/main.tf | 77 ++ .../modules/container_registry/outputs.tf | 29 + .../modules/container_registry/variables.tf | 54 ++ .../modules/deployment_script/main.tf | 95 +++ .../modules/deployment_script/output.tf | 9 + .../modules/deployment_script/variables.tf | 78 ++ .../modules/diagnostic_setting/main.tf | 38 + .../modules/diagnostic_setting/outputs.tf | 9 + .../modules/diagnostic_setting/variables.tf | 79 ++ .../terraform/modules/firewall/main.tf | 310 ++++++++ .../terraform/modules/firewall/outputs.tf | 4 + .../terraform/modules/firewall/variables.tf | 80 ++ .../terraform/modules/key_vault/main.tf | 64 ++ .../terraform/modules/key_vault/outputs.tf | 9 + .../terraform/modules/key_vault/variables.tf | 115 +++ .../terraform/modules/log_analytics/main.tf | 35 + .../terraform/modules/log_analytics/output.tf | 30 + .../modules/log_analytics/variables.tf | 43 + .../terraform/modules/nat_gateway/main.tf | 42 + .../terraform/modules/nat_gateway/output.tf | 14 + .../modules/nat_gateway/variables.tf | 43 + .../modules/network_security_group/main.tf | 58 ++ .../modules/network_security_group/outputs.tf | 4 + .../network_security_group/variables.tf | 36 + .../terraform/modules/node_pool/main.tf | 31 + .../terraform/modules/node_pool/outputs.tf | 4 + .../terraform/modules/node_pool/variables.tf | 144 ++++ .../terraform/modules/openai/main.tf | 79 ++ .../terraform/modules/openai/output.tf | 34 + .../terraform/modules/openai/variables.tf | 70 ++ .../modules/private_dns_zone/main.tf | 26 + .../modules/private_dns_zone/outputs.tf | 4 + .../modules/private_dns_zone/variables.tf | 20 + .../modules/private_endpoint/main.tf | 26 + .../modules/private_endpoint/outputs.tf | 14 + .../modules/private_endpoint/variables.tf | 61 ++ .../terraform/modules/route_table/main.tf | 30 + .../modules/route_table/variables.tf | 35 + .../terraform/modules/storage_account/main.tf | 27 + .../modules/storage_account/outputs.tf | 24 + .../modules/storage_account/variables.tf | 81 ++ .../terraform/modules/virtual_machine/main.tf | 221 ++++++ .../modules/virtual_machine/outputs.tf | 9 + .../modules/virtual_machine/variables.tf | 95 +++ .../terraform/modules/virtual_network/main.tf | 59 ++ .../modules/virtual_network/outputs.tf | 19 + .../modules/virtual_network/variables.tf | 46 ++ .../modules/virtual_network_peering/main.tf | 17 + .../virtual_network_peering/variables.tf | 41 + .../AksOpenAiTerraform/terraform/outputs.tf | 0 .../terraform/register-preview-features.sh | 71 ++ .../terraform/terraform.tfvars | 9 + .../AksOpenAiTerraform/terraform/variables.tf | 743 ++++++++++++++++++ 85 files changed, 6120 insertions(+) create mode 100644 scenarios/AksOpenAiTerraform/README.md create mode 100644 scenarios/AksOpenAiTerraform/scripts/.env create mode 100644 scenarios/AksOpenAiTerraform/scripts/00-variables.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/01-build-docker-image.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/02-run-docker-container.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/03-push-docker-image.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/Dockerfile create mode 100644 scenarios/AksOpenAiTerraform/scripts/app.py create mode 100644 scenarios/AksOpenAiTerraform/scripts/cluster-issuer.yml create mode 100644 scenarios/AksOpenAiTerraform/scripts/configMap.yml create mode 100644 scenarios/AksOpenAiTerraform/scripts/deployment.yml create mode 100644 scenarios/AksOpenAiTerraform/scripts/images/magic8ball.png create mode 100644 scenarios/AksOpenAiTerraform/scripts/images/robot.png create mode 100644 scenarios/AksOpenAiTerraform/scripts/ingress.yml create mode 100644 scenarios/AksOpenAiTerraform/scripts/requirements.txt create mode 100644 scenarios/AksOpenAiTerraform/scripts/service.yml create mode 100644 scenarios/AksOpenAiTerraform/terraform/install-nginx-via-helm-and-create-sa.sh create mode 100644 scenarios/AksOpenAiTerraform/terraform/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/output.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/output.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/node_pool/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/route_table/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/main.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/register-preview-features.sh create mode 100644 scenarios/AksOpenAiTerraform/terraform/terraform.tfvars create mode 100644 scenarios/AksOpenAiTerraform/terraform/variables.tf diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md new file mode 100644 index 000000000..360ebc9b7 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/README.md @@ -0,0 +1,18 @@ +--- +title: How to deploy and run an Azure OpenAI ChatGPT application on AKS via Terraform +description: This article shows how to deploy an AKS cluster and Azure OpenAI Service via Terraform and how to deploy a ChatGPT-like application in Python. +ms.topic: quickstart +ms.date: 09/06/2024 +author: aamini7 +ms.author: ariaamini +ms.custom: innovation-engine, linux-related-content +--- + +## Install AKS extension + +Run commands below to set up AKS extensions for Azure. + +```bash +./terraform/register-preview-features.sh +``` + diff --git a/scenarios/AksOpenAiTerraform/scripts/.env b/scenarios/AksOpenAiTerraform/scripts/.env new file mode 100644 index 000000000..9af98b868 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/.env @@ -0,0 +1,2 @@ +AZURE_OPENAI_TYPE="azure_ad" +AZURE_OPENAI_BASE="https://myopenai.openai.azure.com/" \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/00-variables.sh b/scenarios/AksOpenAiTerraform/scripts/00-variables.sh new file mode 100644 index 000000000..38abccfb6 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/00-variables.sh @@ -0,0 +1,71 @@ +# Variables +acrName="CyanAcr" +acrResourceGrougName="CyanRG" +location="FranceCentral" +attachAcr=false +imageName="magic8ball" +tag="v2" +containerName="magic8ball" +image="$acrName.azurecr.io/$imageName:$tag" +imagePullPolicy="IfNotPresent" # Always, Never, IfNotPresent +managedIdentityName="CyanWorkloadManagedIdentity" +federatedIdentityName="Magic8BallFederatedIdentity" + +# Azure Subscription and Tenant +subscriptionId=$(az account show --query id --output tsv) +subscriptionName=$(az account show --query name --output tsv) +tenantId=$(az account show --query tenantId --output tsv) + +# Parameters +title="Magic 8 Ball" +label="Pose your question and cross your fingers!" +temperature="0.9" +imageWidth="80" + +# OpenAI +openAiName="CyanOpenAi " +openAiResourceGroupName="CyanRG" +openAiType="azure_ad" +openAiBase="https://cyanopenai.openai.azure.com/" +openAiModel="gpt-35-turbo" +openAiDeployment="gpt-35-turbo" + +# Nginx Ingress Controller +nginxNamespace="ingress-basic" +nginxRepoName="ingress-nginx" +nginxRepoUrl="https://kubernetes.github.io/ingress-nginx" +nginxChartName="ingress-nginx" +nginxReleaseName="nginx-ingress" +nginxReplicaCount=3 + +# Certificate Manager +cmNamespace="cert-manager" +cmRepoName="jetstack" +cmRepoUrl="https://charts.jetstack.io" +cmChartName="cert-manager" +cmReleaseName="cert-manager" + +# Cluster Issuer +email="paolos@microsoft.com" +clusterIssuerName="letsencrypt-nginx" +clusterIssuerTemplate="cluster-issuer.yml" + +# AKS Cluster +aksClusterName="CyanAks" +aksResourceGroupName="CyanRG" + +# Sample Application +namespace="magic8ball" +serviceAccountName="magic8ball-sa" +deploymentTemplate="deployment.yml" +serviceTemplate="service.yml" +configMapTemplate="configMap.yml" +secretTemplate="secret.yml" + +# Ingress and DNS +ingressTemplate="ingress.yml" +ingressName="magic8ball-ingress" +dnsZoneName="contoso.com" +dnsZoneResourceGroupName="DnsResourceGroup" +subdomain="magic" +host="$subdomain.$dnsZoneName" \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/01-build-docker-image.sh b/scenarios/AksOpenAiTerraform/scripts/01-build-docker-image.sh new file mode 100644 index 000000000..1425afefb --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/01-build-docker-image.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# For more information, see: +# * https://hub.docker.com/_/python +# * https://docs.streamlit.io/knowledge-base/tutorials/deploy/docker +# * https://stackoverflow.com/questions/30494050/how-do-i-pass-environment-variables-to-docker-containers + +# Variables +source ./00-variables.sh + +# Build the docker image +docker build -t $imageName:$tag -f Dockerfile . \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/02-run-docker-container.sh b/scenarios/AksOpenAiTerraform/scripts/02-run-docker-container.sh new file mode 100644 index 000000000..31e4d7f49 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/02-run-docker-container.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# For more information, see: +# * https://hub.docker.com/_/python +# * https://docs.streamlit.io/knowledge-base/tutorials/deploy/docker +# * https://stackoverflow.com/questions/30494050/how-do-i-pass-environment-variables-to-docker-containers + +# Variables +source ./00-variables.sh + +# Run the docker container +docker run -it \ + --rm \ + -p 8501:8501 \ + -e TEMPERATURE=$temperature \ + -e AZURE_OPENAI_BASE=$AZURE_OPENAI_BASE \ + -e AZURE_OPENAI_KEY=$AZURE_OPENAI_KEY \ + -e AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL \ + -e AZURE_OPENAI_DEPLOYMENT=$AZURE_OPENAI_DEPLOYMENT \ + --name $containerName \ + $imageName:$tag \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/03-push-docker-image.sh b/scenarios/AksOpenAiTerraform/scripts/03-push-docker-image.sh new file mode 100644 index 000000000..e0e9865a9 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/03-push-docker-image.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Variables +source ./00-variables.sh + +# Login to ACR +az acr login --name $acrName + +# Retrieve ACR login server. Each container image needs to be tagged with the loginServer name of the registry. +loginServer=$(az acr show --name $acrName --query loginServer --output tsv) + +# Tag the local image with the loginServer of ACR +docker tag ${imageName,,}:$tag $loginServer/${imageName,,}:$tag + +# Push latest container image to ACR +docker push $loginServer/${imageName,,}:$tag \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh b/scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh new file mode 100644 index 000000000..4e2670847 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Variables +source ./00-variables.sh + +# Use Helm to deploy an NGINX ingress controller +result=$(helm list -n $nginxNamespace | grep $nginxReleaseName | awk '{print $1}') + +if [[ -n $result ]]; then + echo "[$nginxReleaseName] ingress controller already exists in the [$nginxNamespace] namespace" +else + # Check if the ingress-nginx repository is not already added + result=$(helm repo list | grep $nginxRepoName | awk '{print $1}') + + if [[ -n $result ]]; then + echo "[$nginxRepoName] Helm repo already exists" + else + # Add the ingress-nginx repository + echo "Adding [$nginxRepoName] Helm repo..." + helm repo add $nginxRepoName $nginxRepoUrl + fi + + # Update your local Helm chart repository cache + echo 'Updating Helm repos...' + helm repo update + + # Deploy NGINX ingress controller + echo "Deploying [$nginxReleaseName] NGINX ingress controller to the [$nginxNamespace] namespace..." + helm install $nginxReleaseName $nginxRepoName/$nginxChartName \ + --create-namespace \ + --namespace $nginxNamespace \ + --set controller.nodeSelector."kubernetes\.io/os"=linux \ + --set controller.replicaCount=$replicaCount \ + --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz +fi + +# Get values +helm get values $nginxReleaseName --namespace $nginxNamespace diff --git a/scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh b/scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh new file mode 100644 index 000000000..590a41436 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh @@ -0,0 +1,34 @@ +#/bin/bash + +# Variables +source ./00-variables.sh + +# Check if the ingress-nginx repository is not already added +result=$(helm repo list | grep $cmRepoName | awk '{print $1}') + +if [[ -n $result ]]; then + echo "[$cmRepoName] Helm repo already exists" +else + # Add the Jetstack Helm repository + echo "Adding [$cmRepoName] Helm repo..." + helm repo add $cmRepoName $cmRepoUrl +fi + +# Update your local Helm chart repository cache +echo 'Updating Helm repos...' +helm repo update + +# Install cert-manager Helm chart +result=$(helm list -n $cmNamespace | grep $cmReleaseName | awk '{print $1}') + +if [[ -n $result ]]; then + echo "[$cmReleaseName] cert-manager already exists in the $cmNamespace namespace" +else + # Install the cert-manager Helm chart + echo "Deploying [$cmReleaseName] cert-manager to the $cmNamespace namespace..." + helm install $cmReleaseName $cmRepoName/$cmChartName \ + --create-namespace \ + --namespace $cmNamespace \ + --set installCRDs=true \ + --set nodeSelector."kubernetes\.io/os"=linux +fi diff --git a/scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh b/scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh new file mode 100644 index 000000000..fd7976cfb --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh @@ -0,0 +1,19 @@ +#/bin/bash + +# Variables +source ./00-variables.sh + +# Check if the cluster issuer already exists +result=$(kubectl get ClusterIssuer -o json | jq -r '.items[].metadata.name | select(. == "'$clusterIssuerName'")') + +if [[ -n $result ]]; then + echo "[$clusterIssuerName] cluster issuer already exists" + exit +else + # Create the cluster issuer + echo "[$clusterIssuerName] cluster issuer does not exist" + echo "Creating [$clusterIssuerName] cluster issuer..." + cat $clusterIssuerTemplate | + yq "(.spec.acme.email)|="\""$email"\" | + kubectl apply -f - +fi diff --git a/scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh b/scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh new file mode 100644 index 000000000..c770e6476 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# Variables +source ./00-variables.sh + +# Check if the user-assigned managed identity already exists +echo "Checking if [$managedIdentityName] user-assigned managed identity actually exists in the [$aksResourceGroupName] resource group..." + +az identity show \ + --name $managedIdentityName \ + --resource-group $aksResourceGroupName &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$managedIdentityName] user-assigned managed identity actually exists in the [$aksResourceGroupName] resource group" + echo "Creating [$managedIdentityName] user-assigned managed identity in the [$aksResourceGroupName] resource group..." + + # Create the user-assigned managed identity + az identity create \ + --name $managedIdentityName \ + --resource-group $aksResourceGroupName \ + --location $location \ + --subscription $subscriptionId 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$managedIdentityName] user-assigned managed identity successfully created in the [$aksResourceGroupName] resource group" + else + echo "Failed to create [$managedIdentityName] user-assigned managed identity in the [$aksResourceGroupName] resource group" + exit + fi +else + echo "[$managedIdentityName] user-assigned managed identity already exists in the [$aksResourceGroupName] resource group" +fi + +# Retrieve the clientId of the user-assigned managed identity +echo "Retrieving clientId for [$managedIdentityName] managed identity..." +clientId=$(az identity show \ + --name $managedIdentityName \ + --resource-group $aksResourceGroupName \ + --query clientId \ + --output tsv) + +if [[ -n $clientId ]]; then + echo "[$clientId] clientId for the [$managedIdentityName] managed identity successfully retrieved" +else + echo "Failed to retrieve clientId for the [$managedIdentityName] managed identity" + exit +fi + +# Retrieve the principalId of the user-assigned managed identity +echo "Retrieving principalId for [$managedIdentityName] managed identity..." +principalId=$(az identity show \ + --name $managedIdentityName \ + --resource-group $aksResourceGroupName \ + --query principalId \ + --output tsv) + +if [[ -n $principalId ]]; then + echo "[$principalId] principalId for the [$managedIdentityName] managed identity successfully retrieved" +else + echo "Failed to retrieve principalId for the [$managedIdentityName] managed identity" + exit +fi + +# Get the resource id of the Azure OpenAI resource +openAiId=$(az cognitiveservices account show \ + --name $openAiName \ + --resource-group $openAiResourceGroupName \ + --query id \ + --output tsv) + +if [[ -n $openAiId ]]; then + echo "Resource id for the [$openAiName] Azure OpenAI resource successfully retrieved" +else + echo "Failed to the resource id for the [$openAiName] Azure OpenAI resource" + exit -1 +fi + +# Assign the Cognitive Services User role on the Azure OpenAI resource to the managed identity +role="Cognitive Services User" +echo "Checking if the [$managedIdentityName] managed identity has been assigned to [$role] role with [$openAiName] Azure OpenAI resource as a scope..." +current=$(az role assignment list \ + --assignee $principalId \ + --scope $openAiId \ + --query "[?roleDefinitionName=='$role'].roleDefinitionName" \ + --output tsv 2>/dev/null) + +if [[ $current == $role ]]; then + echo "[$managedIdentityName] managed identity is already assigned to the ["$current"] role with [$openAiName] Azure OpenAI resource as a scope" +else + echo "[$managedIdentityName] managed identity is not assigned to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" + echo "Assigning the [$role] role to the [$managedIdentityName] managed identity with [$openAiName] Azure OpenAI resource as a scope..." + + az role assignment create \ + --assignee $principalId \ + --role "$role" \ + --scope $openAiId 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$managedIdentityName] managed identity successfully assigned to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" + else + echo "Failed to assign the [$managedIdentityName] managed identity to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" + exit + fi +fi \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh b/scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh new file mode 100644 index 000000000..5a89a0619 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +# Variables for the user-assigned managed identity +source ./00-variables.sh + +# Check if the namespace already exists +result=$(kubectl get namespace -o 'jsonpath={.items[?(@.metadata.name=="'$namespace'")].metadata.name'}) + +if [[ -n $result ]]; then + echo "[$namespace] namespace already exists" +else + # Create the namespace for your ingress resources + echo "[$namespace] namespace does not exist" + echo "Creating [$namespace] namespace..." + kubectl create namespace $namespace +fi + +# Check if the service account already exists +result=$(kubectl get sa -n $namespace -o 'jsonpath={.items[?(@.metadata.name=="'$serviceAccountName'")].metadata.name'}) + +if [[ -n $result ]]; then + echo "[$serviceAccountName] service account already exists" +else + # Retrieve the resource id of the user-assigned managed identity + echo "Retrieving clientId for [$managedIdentityName] managed identity..." + managedIdentityClientId=$(az identity show \ + --name $managedIdentityName \ + --resource-group $aksResourceGroupName \ + --query clientId \ + --output tsv) + + if [[ -n $managedIdentityClientId ]]; then + echo "[$managedIdentityClientId] clientId for the [$managedIdentityName] managed identity successfully retrieved" + else + echo "Failed to retrieve clientId for the [$managedIdentityName] managed identity" + exit + fi + + # Create the service account + echo "[$serviceAccountName] service account does not exist" + echo "Creating [$serviceAccountName] service account..." + cat </dev/null + +if [[ $? != 0 ]]; then + echo "No [$federatedIdentityName] federated identity credential actually exists in the [$aksResourceGroupName] resource group" + + # Get the OIDC Issuer URL + aksOidcIssuerUrl="$(az aks show \ + --only-show-errors \ + --name $aksClusterName \ + --resource-group $aksResourceGroupName \ + --query oidcIssuerProfile.issuerUrl \ + --output tsv)" + + # Show OIDC Issuer URL + if [[ -n $aksOidcIssuerUrl ]]; then + echo "The OIDC Issuer URL of the $aksClusterName cluster is $aksOidcIssuerUrl" + fi + + echo "Creating [$federatedIdentityName] federated identity credential in the [$aksResourceGroupName] resource group..." + + # Establish the federated identity credential between the managed identity, the service account issuer, and the subject. + az identity federated-credential create \ + --name $federatedIdentityName \ + --identity-name $managedIdentityName \ + --resource-group $aksResourceGroupName \ + --issuer $aksOidcIssuerUrl \ + --subject system:serviceaccount:$namespace:$serviceAccountName + + if [[ $? == 0 ]]; then + echo "[$federatedIdentityName] federated identity credential successfully created in the [$aksResourceGroupName] resource group" + else + echo "Failed to create [$federatedIdentityName] federated identity credential in the [$aksResourceGroupName] resource group" + exit + fi +else + echo "[$federatedIdentityName] federated identity credential already exists in the [$aksResourceGroupName] resource group" +fi \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh b/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh new file mode 100644 index 000000000..3843f71b7 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Variables +source ./00-variables.sh + +# Attach ACR to AKS cluster +if [[ $attachAcr == true ]]; then + echo "Attaching ACR $acrName to AKS cluster $aksClusterName..." + az aks update \ + --name $aksClusterName \ + --resource-group $aksResourceGroupName \ + --attach-acr $acrName +fi + +# Check if namespace exists in the cluster +result=$(kubectl get namespace -o jsonpath="{.items[?(@.metadata.name=='$namespace')].metadata.name}") + +if [[ -n $result ]]; then + echo "$namespace namespace already exists in the cluster" +else + echo "$namespace namespace does not exist in the cluster" + echo "creating $namespace namespace in the cluster..." + kubectl create namespace $namespace +fi + +# Create config map +cat $configMapTemplate | + yq "(.data.TITLE)|="\""$title"\" | + yq "(.data.LABEL)|="\""$label"\" | + yq "(.data.TEMPERATURE)|="\""$temperature"\" | + yq "(.data.IMAGE_WIDTH)|="\""$imageWidth"\" | + yq "(.data.AZURE_OPENAI_TYPE)|="\""$openAiType"\" | + yq "(.data.AZURE_OPENAI_BASE)|="\""$openAiBase"\" | + yq "(.data.AZURE_OPENAI_MODEL)|="\""$openAiModel"\" | + yq "(.data.AZURE_OPENAI_DEPLOYMENT)|="\""$openAiDeployment"\" | + kubectl apply -n $namespace -f - + +# Create deployment +cat $deploymentTemplate | + yq "(.spec.template.spec.containers[0].image)|="\""$image"\" | + yq "(.spec.template.spec.containers[0].imagePullPolicy)|="\""$imagePullPolicy"\" | + yq "(.spec.template.spec.serviceAccountName)|="\""$serviceAccountName"\" | + kubectl apply -n $namespace -f - + +# Create deployment +kubectl apply -f $serviceTemplate -n $namespace \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh b/scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh new file mode 100644 index 000000000..388518355 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh @@ -0,0 +1,12 @@ +#/bin/bash + +# Variables +source ./00-variables.sh + +# Create the ingress +echo "[$ingressName] ingress does not exist" +echo "Creating [$ingressName] ingress..." +cat $ingressTemplate | + yq "(.spec.tls[0].hosts[0])|="\""$host"\" | + yq "(.spec.rules[0].host)|="\""$host"\" | + kubectl apply -n $namespace -f - \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh b/scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh new file mode 100644 index 000000000..95f8baf69 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh @@ -0,0 +1,79 @@ +# Variables +source ./00-variables.sh + +# Retrieve the public IP address from the ingress +echo "Retrieving the external IP address from the [$ingressName] ingress..." +publicIpAddress=$(kubectl get ingress $ingressName -n $namespace -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + +if [ -n $publicIpAddress ]; then + echo "[$publicIpAddress] external IP address of the application gateway ingress controller successfully retrieved from the [$ingressName] ingress" +else + echo "Failed to retrieve the external IP address of the application gateway ingress controller from the [$ingressName] ingress" + exit +fi + +# Check if an A record for todolist subdomain exists in the DNS Zone +echo "Retrieving the A record for the [$subdomain] subdomain from the [$dnsZoneName] DNS zone..." +ipv4Address=$(az network dns record-set a list \ + --zone-name $dnsZoneName \ + --resource-group $dnsZoneResourceGroupName \ + --query "[?name=='$subdomain'].arecords[].ipv4Address" \ + --output tsv) + +if [[ -n $ipv4Address ]]; then + echo "An A record already exists in [$dnsZoneName] DNS zone for the [$subdomain] subdomain with [$ipv4Address] IP address" + + if [[ $ipv4Address == $publicIpAddress ]]; then + echo "The [$ipv4Address] ip address of the existing A record is equal to the ip address of the [$ingressName] ingress" + echo "No additional step is required" + exit + else + echo "The [$ipv4Address] ip address of the existing A record is different than the ip address of the [$ingressName] ingress" + fi + + # Retrieving name of the record set relative to the zone + echo "Retrieving the name of the record set relative to the [$dnsZoneName] zone..." + + recordSetName=$(az network dns record-set a list \ + --zone-name $dnsZoneName \ + --resource-group $dnsZoneResourceGroupName \ + --query "[?name=='$subdomain'].name" \ + --output name 2>/dev/null) + + if [[ -n $recordSetName ]]; then + "[$recordSetName] record set name successfully retrieved" + else + "Failed to retrieve the name of the record set relative to the [$dnsZoneName] zone" + exit + fi + + # Remove the a record + echo "Removing the A record from the record set relative to the [$dnsZoneName] zone..." + + az network dns record-set a remove-record \ + --ipv4-address $ipv4Address \ + --record-set-name $recordSetName \ + --zone-name $dnsZoneName \ + --resource-group $dnsZoneResourceGroupName + + if [[ $? == 0 ]]; then + echo "[$ipv4Address] ip address successfully removed from the [$recordSetName] record set" + else + echo "Failed to remove the [$ipv4Address] ip address from the [$recordSetName] record set" + exit + fi +fi + +# Create the a record +echo "Creating an A record in [$dnsZoneName] DNS zone for the [$subdomain] subdomain with [$publicIpAddress] IP address..." +az network dns record-set a add-record \ + --zone-name $dnsZoneName \ + --resource-group $dnsZoneResourceGroupName \ + --record-set-name $subdomain \ + --ipv4-address $publicIpAddress 1>/dev/null + +if [[ $? == 0 ]]; then + echo "A record for the [$subdomain] subdomain with [$publicIpAddress] IP address successfully created in [$dnsZoneName] DNS zone" +else + echo "Failed to create an A record for the $subdomain subdomain with [$publicIpAddress] IP address in [$dnsZoneName] DNS zone" +fi diff --git a/scenarios/AksOpenAiTerraform/scripts/Dockerfile b/scenarios/AksOpenAiTerraform/scripts/Dockerfile new file mode 100644 index 000000000..2f603014f --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/Dockerfile @@ -0,0 +1,94 @@ +# app/Dockerfile + +# # Stage 1 - Install build dependencies + +# A Dockerfile must start with a FROM instruction which sets the base image for the container. +# The Python images come in many flavors, each designed for a specific use case. +# The python:3.11-slim image is a good base image for most applications. +# It is a minimal image built on top of Debian Linux and includes only the necessary packages to run Python. +# The slim image is a good choice because it is small and contains only the packages needed to run Python. +# For more information, see: +# * https://hub.docker.com/_/python +# * https://docs.streamlit.io/knowledge-base/tutorials/deploy/docker +FROM python:3.11-slim AS builder + +# The WORKDIR instruction sets the working directory for any RUN, CMD, ENTRYPOINT, COPY and ADD instructions that follow it in the Dockerfile. +# If the WORKDIR doesn’t exist, it will be created even if it’s not used in any subsequent Dockerfile instruction. +# For more information, see: https://docs.docker.com/engine/reference/builder/#workdir +WORKDIR /app + +# Set environment variables. +# The ENV instruction sets the environment variable to the value . +# This value will be in the environment of all “descendant” Dockerfile commands and can be replaced inline in many as well. +# For more information, see: https://docs.docker.com/engine/reference/builder/#env +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# Install git so that we can clone the app code from a remote repo using the RUN instruction. +# The RUN comand has 2 forms: +# * RUN (shell form, the command is run in a shell, which by default is /bin/sh -c on Linux or cmd /S /C on Windows) +# * RUN ["executable", "param1", "param2"] (exec form) +# The RUN instruction will execute any commands in a new layer on top of the current image and commit the results. +# The resulting committed image will be used for the next step in the Dockerfile. +# For more information, see: https://docs.docker.com/engine/reference/builder/#run +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + software-properties-common \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Create a virtualenv to keep dependencies together +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +# Clone the requirements.txt which contains dependencies to WORKDIR +# COPY has two forms: +# * COPY (this copies the files from the local machine to the container's own filesystem) +# * COPY ["",... ""] (this form is required for paths containing whitespace) +# For more information, see: https://docs.docker.com/engine/reference/builder/#copy +COPY requirements.txt . + +# Install the Python dependencies +RUN pip install --no-cache-dir --no-deps -r requirements.txt + +# Stage 2 - Copy only necessary files to the runner stage + +# The FROM instruction initializes a new build stage for the application +FROM python:3.11-slim + +# Sets the working directory to /app +WORKDIR /app + +# Copy the virtual environment from the builder stage +COPY --from=builder /opt/venv /opt/venv + +# Set environment variables +ENV PATH="/opt/venv/bin:$PATH" + +# Clone the app.py containing the application code +COPY app.py . + +# Copy the images folder to WORKDIR +# The ADD instruction copies new files, directories or remote file URLs from and adds them to the filesystem of the image at the path . +# For more information, see: https://docs.docker.com/engine/reference/builder/#add +ADD images ./images + +# The EXPOSE instruction informs Docker that the container listens on the specified network ports at runtime. +# For more information, see: https://docs.docker.com/engine/reference/builder/#expose +EXPOSE 8501 + +# The HEALTHCHECK instruction has two forms: +# * HEALTHCHECK [OPTIONS] CMD command (check container health by running a command inside the container) +# * HEALTHCHECK NONE (disable any healthcheck inherited from the base image) +# The HEALTHCHECK instruction tells Docker how to test a container to check that it is still working. +# This can detect cases such as a web server that is stuck in an infinite loop and unable to handle new connections, +# even though the server process is still running. For more information, see: https://docs.docker.com/engine/reference/builder/#healthcheck +HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health + +# The ENTRYPOINT instruction has two forms: +# * ENTRYPOINT ["executable", "param1", "param2"] (exec form, preferred) +# * ENTRYPOINT command param1 param2 (shell form) +# The ENTRYPOINT instruction allows you to configure a container that will run as an executable. +# For more information, see: https://docs.docker.com/engine/reference/builder/#entrypoint +ENTRYPOINT ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"] \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/app.py b/scenarios/AksOpenAiTerraform/scripts/app.py new file mode 100644 index 000000000..4211c57ca --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/app.py @@ -0,0 +1,347 @@ +""" +MIT License + +Copyright (c) 2023 Paolo Salvatori + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +# This sample is based on the following article: +# +# - https://levelup.gitconnected.com/its-time-to-create-a-private-chatgpt-for-yourself-today-6503649e7bb6 +# +# Use pip to install the following packages: +# +# - streamlit +# - openai +# - streamlit-chat +# - azure.identity +# - dotenv +# +# Make sure to provide a value for the following environment variables: +# +# - AZURE_OPENAI_BASE: the URL of your Azure OpenAI resource, for example https://eastus.api.cognitive.microsoft.com/ +# - AZURE_OPENAI_KEY: the key of your Azure OpenAI resource +# - AZURE_OPENAI_DEPLOYMENT: the name of the ChatGPT deployment used by your Azure OpenAI resource +# - AZURE_OPENAI_MODEL: the name of the ChatGPT model used by your Azure OpenAI resource, for example gpt-35-turbo +# - TITLE: the title of the Streamlit app +# - TEMPERATURE: the temperature used by the OpenAI API to generate the response +# - SYSTEM: give the model instructions about how it should behave and any context it should reference when generating a response. +# Used to describe the assistant's personality. +# +# You can use two different authentication methods: +# +# - API key: set the AZURE_OPENAI_TYPE environment variable to azure and the AZURE_OPENAI_KEY environment variable to the key of +# your Azure OpenAI resource. You can use the regional endpoint, such as https://eastus.api.cognitive.microsoft.com/, passed in +# the AZURE_OPENAI_BASE environment variable, to connect to the Azure OpenAI resource. +# - Azure Active Directory: set the AZURE_OPENAI_TYPE environment variable to azure_ad and use a service principal or managed +# identity with the DefaultAzureCredential object to acquire a token. For more information on the DefaultAzureCredential in Python, +# see https://docs.microsoft.com/en-us/azure/developer/python/azure-sdk-authenticate?tabs=cmd +# Make sure to assign the "Cognitive Services User" role to the service principal or managed identity used to authenticate to +# Azure OpenAI. For more information, see https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/managed-identity. +# If you want to use Azure AD integrated security, you need to create a custom subdomain for your Azure OpenAI resource and use the +# specific endpoint containing the custom domain, such as https://bingo.openai.azure.com/ where bingo is the custom subdomain. +# If you specify the regional endpoint, you get a wonderful error: "Subdomain does not map to a resource.". +# Hence, make sure to pass the endpoint containing the custom domain in the AZURE_OPENAI_BASE environment variable. +# +# Use the following command to run the app: +# +# - streamlit run app.py + +# Import packages +import os +import sys +import time +import openai +import logging +import streamlit as st +from streamlit_chat import message +from azure.identity import DefaultAzureCredential +from dotenv import load_dotenv +from dotenv import dotenv_values + +# Load environment variables from .env file +if os.path.exists(".env"): + load_dotenv(override=True) + config = dotenv_values(".env") + +# Read environment variables +assistan_profile = """ +You are the infamous Magic 8 Ball. You need to randomly reply to any question with one of the following answers: + +- It is certain. +- It is decidedly so. +- Without a doubt. +- Yes definitely. +- You may rely on it. +- As I see it, yes. +- Most likely. +- Outlook good. +- Yes. +- Signs point to yes. +- Reply hazy, try again. +- Ask again later. +- Better not tell you now. +- Cannot predict now. +- Concentrate and ask again. +- Don't count on it. +- My reply is no. +- My sources say no. +- Outlook not so good. +- Very doubtful. + +Add a short comment in a pirate style at the end! Follow your heart and be creative! +For mor information, see https://en.wikipedia.org/wiki/Magic_8_Ball +""" +title = os.environ.get("TITLE", "Magic 8 Ball") +text_input_label = os.environ.get("TEXT_INPUT_LABEL", "Pose your question and cross your fingers!") +image_file_name = os.environ.get("IMAGE_FILE_NAME", "magic8ball.png") +image_width = int(os.environ.get("IMAGE_WIDTH", 80)) +temperature = float(os.environ.get("TEMPERATURE", 0.9)) +system = os.environ.get("SYSTEM", assistan_profile) +api_base = os.getenv("AZURE_OPENAI_BASE") +api_key = os.getenv("AZURE_OPENAI_KEY") +api_type = os.environ.get("AZURE_OPENAI_TYPE", "azure") +api_version = os.environ.get("AZURE_OPENAI_VERSION", "2023-05-15") +engine = os.getenv("AZURE_OPENAI_DEPLOYMENT") +model = os.getenv("AZURE_OPENAI_MODEL") + +# Configure OpenAI +openai.api_type = api_type +openai.api_version = api_version +openai.api_base = api_base + +# Set default Azure credential +default_credential = DefaultAzureCredential() if openai.api_type == "azure_ad" else None + +# Configure a logger +logging.basicConfig(stream = sys.stdout, + format = '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s', + level = logging.INFO) +logger = logging.getLogger(__name__) + +# Log variables +logger.info(f"title: {title}") +logger.info(f"text_input_label: {text_input_label}") +logger.info(f"image_file_name: {image_file_name}") +logger.info(f"image_width: {image_width}") +logger.info(f"temperature: {temperature}") +logger.info(f"system: {system}") +logger.info(f"api_base: {api_base}") +logger.info(f"api_key: {api_key}") +logger.info(f"api_type: {api_type}") +logger.info(f"api_version: {api_version}") +logger.info(f"engine: {engine}") +logger.info(f"model: {model}") + +# Authenticate to Azure OpenAI +if openai.api_type == "azure": + openai.api_key = api_key +elif openai.api_type == "azure_ad": + openai_token = default_credential.get_token("https://cognitiveservices.azure.com/.default") + openai.api_key = openai_token.token + if 'openai_token' not in st.session_state: + st.session_state['openai_token'] = openai_token +else: + logger.error("Invalid API type. Please set the AZURE_OPENAI_TYPE environment variable to azure or azure_ad.") + raise ValueError("Invalid API type. Please set the AZURE_OPENAI_TYPE environment variable to azure or azure_ad.") + +# Customize Streamlit UI using CSS +st.markdown(""" + +""", unsafe_allow_html=True) + +# Initialize Streamlit session state +if 'prompts' not in st.session_state: + st.session_state['prompts'] = [{"role": "system", "content": system}] + +if 'generated' not in st.session_state: + st.session_state['generated'] = [] + +if 'past' not in st.session_state: + st.session_state['past'] = [] + +# Refresh the OpenAI security token every 45 minutes +def refresh_openai_token(): + if st.session_state['openai_token'].expires_on < int(time.time()) - 45 * 60: + st.session_state['openai_token'] = default_credential.get_token("https://cognitiveservices.azure.com/.default") + openai.api_key = st.session_state['openai_token'].token + +# Send user prompt to Azure OpenAI +def generate_response(prompt): + try: + st.session_state['prompts'].append({"role": "user", "content": prompt}) + + if openai.api_type == "azure_ad": + refresh_openai_token() + + completion = openai.ChatCompletion.create( + engine = engine, + model = model, + messages = st.session_state['prompts'], + temperature = temperature, + ) + + message = completion.choices[0].message.content + return message + except Exception as e: + logging.exception(f"Exception in generate_response: {e}") + +# Reset Streamlit session state to start a new chat from scratch +def new_click(): + st.session_state['prompts'] = [{"role": "system", "content": system}] + st.session_state['past'] = [] + st.session_state['generated'] = [] + st.session_state['user'] = "" + +# Handle on_change event for user input +def user_change(): + # Avoid handling the event twice when clicking the Send button + chat_input = st.session_state['user'] + st.session_state['user'] = "" + if (chat_input == '' or + (len(st.session_state['past']) > 0 and chat_input == st.session_state['past'][-1])): + return + + # Generate response invoking Azure OpenAI LLM + if chat_input != '': + output = generate_response(chat_input) + + # store the output + st.session_state['past'].append(chat_input) + st.session_state['generated'].append(output) + st.session_state['prompts'].append({"role": "assistant", "content": output}) + +# Create a 2-column layout. Note: Streamlit columns do not properly render on mobile devices. +# For more information, see https://github.com/streamlit/streamlit/issues/5003 +col1, col2 = st.columns([1, 7]) + +# Display the robot image +with col1: + st.image(image = os.path.join("images", image_file_name), width = image_width) + +# Display the title +with col2: + st.title(title) + +# Create a 3-column layout. Note: Streamlit columns do not properly render on mobile devices. +# For more information, see https://github.com/streamlit/streamlit/issues/5003 +col3, col4, col5 = st.columns([7, 1, 1]) + +# Create text input in column 1 +with col3: + user_input = st.text_input(text_input_label, key = "user", on_change = user_change) + +# Create send button in column 2 +with col4: + st.button(label = "Send") + +# Create new button in column 3 +with col5: + st.button(label = "New", on_click = new_click) + +# Display the chat history in two separate tabs +# - normal: display the chat history as a list of messages using the streamlit_chat message() function +# - rich: display the chat history as a list of messages using the Streamlit markdown() function +if st.session_state['generated']: + tab1, tab2 = st.tabs(["normal", "rich"]) + with tab1: + for i in range(len(st.session_state['generated']) - 1, -1, -1): + message(st.session_state['past'][i], is_user = True, key = str(i) + '_user', avatar_style = "fun-emoji", seed = "Nala") + message(st.session_state['generated'][i], key = str(i), avatar_style = "bottts", seed = "Fluffy") + with tab2: + for i in range(len(st.session_state['generated']) - 1, -1, -1): + st.markdown(st.session_state['past'][i]) + st.markdown(st.session_state['generated'][i]) \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/cluster-issuer.yml b/scenarios/AksOpenAiTerraform/scripts/cluster-issuer.yml new file mode 100644 index 000000000..6855fdf8c --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/cluster-issuer.yml @@ -0,0 +1,18 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-nginx +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: paolos@microsoft.com + privateKeySecretRef: + name: letsencrypt + solvers: + - http01: + ingress: + class: nginx + podTemplate: + spec: + nodeSelector: + "kubernetes.io/os": linux \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/configMap.yml b/scenarios/AksOpenAiTerraform/scripts/configMap.yml new file mode 100644 index 000000000..fb668c832 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/configMap.yml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: magic8ball-configmap +data: + TITLE: "Magic 8 Ball" + LABEL: "Pose your question and cross your fingers!" + TEMPERATURE: "0.9" + IMAGE_WIDTH: "80" + AZURE_OPENAI_TYPE: azure_ad + AZURE_OPENAI_BASE: https://myopenai.openai.azure.com/ + AZURE_OPENAI_KEY: "" + AZURE_OPENAI_MODEL: gpt-35-turbo + AZURE_OPENAI_DEPLOYMENT: magic8ballGPT diff --git a/scenarios/AksOpenAiTerraform/scripts/deployment.yml b/scenarios/AksOpenAiTerraform/scripts/deployment.yml new file mode 100644 index 000000000..afffab8df --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/deployment.yml @@ -0,0 +1,123 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: magic8ball + labels: + app: magic8ball +spec: + replicas: 3 + selector: + matchLabels: + app: magic8ball + azure.workload.identity/use: "true" + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + minReadySeconds: 5 + template: + metadata: + labels: + app: magic8ball + azure.workload.identity/use: "true" + prometheus.io/scrape: "true" + spec: + serviceAccountName: magic8ball-sa + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: magic8ball + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: magic8ball + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: magic8ball + image: paolosalvatori.azurecr.io/magic8ball:v1 + imagePullPolicy: Always + resources: + requests: + memory: "128Mi" + cpu: "250m" + limits: + memory: "256Mi" + cpu: "500m" + ports: + - containerPort: 8501 + livenessProbe: + httpGet: + path: / + port: 8501 + failureThreshold: 1 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 8501 + failureThreshold: 1 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + startupProbe: + httpGet: + path: / + port: 8501 + failureThreshold: 1 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + env: + - name: TITLE + valueFrom: + configMapKeyRef: + name: magic8ball-configmap + key: TITLE + - name: IMAGE_WIDTH + valueFrom: + configMapKeyRef: + name: magic8ball-configmap + key: IMAGE_WIDTH + - name: LABEL + valueFrom: + configMapKeyRef: + name: magic8ball-configmap + key: LABEL + - name: TEMPERATURE + valueFrom: + configMapKeyRef: + name: magic8ball-configmap + key: TEMPERATURE + - name: AZURE_OPENAI_TYPE + valueFrom: + configMapKeyRef: + name: magic8ball-configmap + key: AZURE_OPENAI_TYPE + - name: AZURE_OPENAI_BASE + valueFrom: + configMapKeyRef: + name: magic8ball-configmap + key: AZURE_OPENAI_BASE + - name: AZURE_OPENAI_KEY + valueFrom: + configMapKeyRef: + name: magic8ball-configmap + key: AZURE_OPENAI_KEY + - name: AZURE_OPENAI_MODEL + valueFrom: + configMapKeyRef: + name: magic8ball-configmap + key: AZURE_OPENAI_MODEL + - name: AZURE_OPENAI_DEPLOYMENT + valueFrom: + configMapKeyRef: + name: magic8ball-configmap + key: AZURE_OPENAI_DEPLOYMENT \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/images/magic8ball.png b/scenarios/AksOpenAiTerraform/scripts/images/magic8ball.png new file mode 100644 index 0000000000000000000000000000000000000000..cd53753774ed4e666c7093f6d58ca02a25be36a1 GIT binary patch literal 37452 zcmW(+1yEaUv&IP?+#$HTI|O$v?ozC1ahKrkTHIRPiWDzyrC5s=_YW@)z30D~$s}`T zk~x#?yZhM2X=x~8p_8G*!NFlEE6M4=!NIft?+HQxuG~?R(gH8+Uh;-svYvKUUXE_A z5Isj1J2(YTM+mPVMBU0C!pqIe2jLbG<>MFS=Yg|VxholcgoDGt``;6Oek0@sxQJvgqb>sn*PMd!VucI`msYMUC!^=LdKUcn zG3%HAzpmfP6w;t@`nG0JGYB4G`n_15u+NT9e&GQEGi-!PDoUaMy*K!6^#-|!cBFAl#g zs~yWDLx!VYXUHQY#NmJ_eHYN0BxA^V0Iw}hJ{wqC44>j`d?-vW2Qb~WRivrLO z$;rsb$jKk;APW+HI@$mG79HZjM2I~rNB{H21(mp8*q=XdcO9V*->Kg+iL7yug@<3^ z(%JC-=0Ww5QeTj`AG>}8pWhV!dm)~0;bjvxFDAm$UhiN3bi2Z;DQ$R@Zi_rD!N-T# z6MQ|vG_ZCN`Y#ZElY0KTx{loz`S&jTpYJv@lvh>#I+uj;as*Vn#ACx1kedXHmmR(z z;*o!rY7IK7kf{SACMsE?m~;u+nGwjfE`R_2{qXR>)mJ)& zoK8$k93MM|Vby#K{r5KJSbBuNfj2Tl5=97+fgiCUC?ZCSMr>4wucc*`dWXz(B@g{% zzehz|?8f*jH3hGbAhA}z9)aNW^fY{;s)`Y$yp21xv7e&sk1#YslrG79yak6#Vzvc#c(E}qLB!8{nYtm^ zcxh;0J&9gLh{wXh;)eWcF#BfpJvuoVyG%Rr2UBGWP!WVgx8yK1>=G9ZH8r(rK~w4nK zFp(8>Dw%`QkICGgCWu-8K_~ zexOksPIg3c)3E=zg5g)+1!xZ5?3Vb_`#d{%UK;Y1%OrB4^OaWAoXaAnq9t*l6l~;& zF}yT#g9bLIr!g-Xvk9w-q$5hCln_%P!w6xhxbSC=^s)X`Vy07^GF5&mZD!R>NA0&C zlVp?Be9^Tp9W%|H{3Z8g{zDy-a1a=#n^U4elP*i@Q4}+{N}vi;I#Wqd-ao|WSD3^H zKVc|;3<@gzT~JVfc;|58AxZg)QjVSKtfXpVLzgTQ%gey*lQT|Bc>&$FiT`dfJVxix z>^`NKI22bDlR9aV@3fcR_a)0v)p(NON$=RT@|ienn5!H4K8zF|R;-GV{Fwj_`2S44 zekeV89(M4W!UwNLk?kHGF;`HlNybXft+*{9=pd4YFW`qjxRj&}1NbxG#6@Y~F|`lt zy2P-QFtj%<`GZ-!z`S(o5VBcks}u4%zOpg&5awlOa(MYBrr8MIld#by!+z$b7%LvA z$e5e6GaJ+U?7%|mGN2^?8H^vet#^9d9L!8iC>sVt*m?FG9_P?xc2QM`XLHq@2~%n& zl=Kw?yGLLic4d&;zPX1*?XDGJ+FuDm1~n}p?E0HUzMoD5DSTxQ`-#le6pt7bP}cb< z#Lpm#uN@r=G;QlPZ(5m5kQWP(Oi*|_3My($baY9??2jJ{h=Vu7RKZsd$QIRsS`lB` zZ)DrFwB^*{v|j?0AEKxxQtZvH$0rIuDa0||Q57U&%lz8t`WO>7YynA<`O-!OFJE&$ zu$|o|&Ov!_uu-ns)%d|sBIEI{SbWZYcB&KlgkQl5vIkXP0i1lepmCA`l#q}1{ z3k1D^hnLqJ^|$r5rVsnA7|`fA)4m{51aLmnD`#d1|4E5^gj_&CfO~31ACYu&2W~O% z$5dNw7G4=mX=BfMc|AF1qa1dmlfb?7sD{{*)}}8%0zI5JJ{11;Frz;C55}yg+b|ko z37b?=e(=&C*b=V%xv_yI9rC`KUoruLBmn1#h=>>*9Q^(J_Y#~*AqdzDkrm!;^Q#N_ z8Icy+JN9(q6vx9Y3C!noG|{{#d?8B^%-nD^s^b2O0j0o*Qmgd&NYGx?IV)EA@<-6O zpUu2y%rE=XlHCZD$bv#VJn;Nwnk1CTwJ+es=exGB{q3CXz$tY)RbxpLh^^f3$D2c- zP6qQh^EqMYHSnhXK*h{u%FQ{hj0g+-9JcK(hUt^H=MM_2l0ro*u4I@iGF9q(xP7RF z;6G}I{KZg*yHN<&DoW4DhBiE_E7lZbzQ#=?@!ggncr*UJ8qh*|K0P~w)j#CvXi2d_ zY`5NphDvP-2?%)KpKtuBMz#&Fg+$FbmTr$hNN~dfoYrhL^bm9w3bl%|Ll`@X7asPq zU|)~F8X9DTt&$GCqln=|s7FoGgkAs6*W)+({RO*Zgo{8Aq|hXViT=i0LByN!m6qLo zTy_hL&Pr*S672sXqvY@X->QJdD|`rEOX63=kgYxvw?qjg_!q;SDcJDH$Pkp2WPU#p zu_zVh;XxAaP=?fk#@xnO7o?!4dpam=+N;@Yh)-f}tD~si0($3Aof=O<*}!@*i8pIs zXVD)rmQ2~(+l%>R;!69)#eZ4`RDo5E3<8Rk@udU<-hjkWbP|3jIvRy%o1JZzS$>gi z!PGYvC6O(8PeWJ5`&QEKxMO*(hl>Y2X45zNe|PKe%OjFv40 zMaj6po2e}(bAw3-_A-3p$3)y&sZp(s8Bc`vT5=D2V6N(csn!$Ut(9wr5r}+9AdUic^k#s@xQ2h6# zoco=*IkbYM1z#dRp}m3LA!}uF>!Bd-V?`lQ?zg`hgpmvkR`29u1*a% z_DG;a!zfL_#j~t;vT3{&xhPk3v4=Esb33D1qmox#mP3mX7A7-6-sjIOi8=JUrFuw= zR&(SI%T4_Auk&! zI+35@?b$~iQ#C_^aphetrwN^gt?T9CCwE$gflr>CdSHooPYR_7F;5Wl1H2U~MP;80Oeju0x#k!4lk zK@TqLow&QhhNP0#QIrLXWc;Kq{w_mcgi6@aQf9i;lLT-ojy5HZCHMO(*pU=Y+}cIo z>MMj_)TxzT5dV1+;X3RkTI=fKYmLq65{!oKq8O7tL`|ZLg#vxdRQE16Tag4tf(W@h zTWOVs+ywDXBOeaR!S~mjbO;Ix>L6#Gb<>`<`RrkFX5d8a6?sQ`90^p2<_B4i!Ahrc z{N&s2joiw2mbfLiqTaIWqC5V0DRjxjDHb8|cHKqv;6W-PlMYF_WXch>V5LeItL27N z@4#jEjb`6aNR^10bveL}00sz0UtL;i2M%V#o5rFsDyg5_d3G%SB~=0Qd?F6o=8uu$ zc)bY>zWbHAmN@Y+mJY|Xdy^4ldPI*T-e2%iAX#qR7tc~sVJ_K$yuD<|@I^QxD)-Km zdTv&xA|A{#{_saJD#&f4J1T7;OlA!UWVH$(qL@PE;_5mIAr&~awBv~nn^Zl2a?;@C zzL^u9Zw7fPUqC zw(>`xsz^ANf6=)$X&84G_YM9>>W05JP>W)6fwLsl=J2~7uU%J}00CPy4W#Dz%Ea-H z-eekWALR7&?e#ej7`BI=3y!4w+6Xkn&cz}hQ4}D=zf|7Skc0!hHU@_p1tAYXPcU%K z=oQZbE(ng4^54PgnG1x;ns22a=@vRxbUgs8F~@F=Vx0urRX)&i>m(t zYvE+3ljRXOYYgBWh{};OU)kRB2$ZZpoYJw%u+3~Y^Y$j6t)iyoT zyeZX18N=;tEc#*aS& zNe1V3!ejlI#vZ#95fxkdgX9b24jmknWXJwT5hVqM?VU49d1)Bkxh4hyj*3(w)-UCw zAp9Yf40j%F@7cYkNi_mKMpTZ2Al6y-L>$Ms-B}MYks%q{zxU?x_qH{#d^+<5`kC4k zi=jl3PVU=bnpx?m?Hp_okn(!;4Y!V{m>Bi-N4=Uq=1>ii^S#_8PI^}M z=p|yu7QZpkk~c#7u`DOCZnyAv!ur;$Y0<9(qoPxnepV2o-`<~zx)k)Zz$ z>iRoW%s*t-RdnQSqJQJ+pt?0j56-^jt9FpYy`p4DH)>QD7@+gZT>lR+2 z2G7a7=|m2DbmvSNE{UUYMJSZVql&;a2`A8XW<#~K;iHLF{Ee$NRf9!86)WB}QCKP~ z&aRuK1hmo`Bz#7kI;Vr{xp_So4wmqf;3zLI_k!WUj^?VWs_ct{?*_sVk38;JsFe$? zKM85m9Q3YKiU={}p)F(w3LsRsq!N7&eXE*~>H1k+>ej{N?e@)rOGG*?+@`q&=cB?2 zAVSc}(fT4Y5umA#x|w!+#c?jgxSZ^4BW%Au|F6p~|N71WpZQ+0d$-=m-9 zbI@Y{1JZTI6!*5Cg24GXW~FpaDgA^7PTh0BhMrebv@>|mJ=jgq_b-Oe8z9PBgR{bR$y1^x7xgOcLL z+XPr&ChTTsXUoQ5d;9u+)J`w5_$qjmu*(e`60O!Mb+rCG0K zjw-e4vJ3Y9h58#x$G%0Nqmzyw`zg;Lwc~dW4@RQ;UTk6^@tTs0qLeF`c?!WZeA(kZ z>>?<$#Fw^I2Iq56C%QS-^{^f?=$@HkO~~R$sZ(gX{^&%+?UwSIs5d`~!n2s9jItY) z+@tA5F|}!ST4F7-!ep&(?7OYxlJPZj9s2j@S}P2Eb~2ho4m3qdDysL)n9@o-v3-$5 z=&$gv<96XkG7}x|;6du2t$eurJ9t`51zdv-p+8N|4^`%SuS$Qpw4=eX&)-dgNj}?F z@X5NxN~PEA(Q&?xPd8DqmnDBz_s>c!;Cr*Y>V)rJipC<(&bs^8Ax;3Hprnk#q#z?J zV~UPv9)>)fj;GNzRSY&Gg)?`m%;r$Tp1z=^>b>abh@%lbWxuO@5Tko4y&9Z;HoH$> z(s5gx*%>&luf9)ME>I_MEPY}cdEeh-vCgNKG$>AdW;pr%>FJ4nN!5c&59!3(5=$fG zVLK)@=7;zV=DCr^Nhpal2&qWe!#X6S=<(+qR~Yp>kVEO#JGqjx?}fJTE2C+!IL%{T zY-GO#fvK~ZzXU5krlYKn(nH=clk0@|?e!Bt{gk;$sJZbhma~$ zs%@>c^TBe;y(sTz@}kn`zMbib3?`4Y?e8%;23%(E!J9q)H@C+Ne0+SZt*z_Dm}aBx zxMpT%O+|oOL4)IHSP=FZzq)DMA_GyIQ91(GnCm_r&Cj8S(5E2pMyKL!b9JvDQ|i7+q&$W0e?V8$4NoVGY<9X zY8+jbIQ#rr)%<$-kDPx}VrIu%>DsNuy$)zJpBuWn-)qYeycjjn^Q+ECmbL7A3O4c! z@&YP^iIFk44JG==d}^;rHWqAmcbAyU9L6~rg$qY@vsCW;)$I>9R%vT%#zFCiP&P=L z(j-iAiY(BVuy~P)NhVOLo-1necv+YD^lDQ2NPjdFaVY%{>&mt{cjgHDWbovg&!vI*9ZXK@Dm4w?&(ylGQvIk*(rsceF49AilRrqRsK}5@{YIz8(*T%j> z5_0|`6Fg`iVd+YI+DHKwgfb1*5iS+TOCEj*^tEBIZHIMpRr#P^X1$B885 ze zGpa=_KOFA_iz8Hn-qXqzwq7#|&F=stq$y5Mcs{Q*WAhyA9lIp+X7iLT+f}}&gwTJo zB;h5r?m^5Jn9m0hmhPd2UH&(W_u)=1E@mon>;fPfxJJ`11ZP8^+U!Dt|Ni^$`Tj!D z7cObugV6-S6<2VJSxWU4)P4-VFm2fKsI70Wc48FRoDHslcDikw=0Bq3(O=s;y1=TXG%RJ=8&rd z3tJsQcb_n<;n78_P9!H-K8kXzVm7X+4J+-#5e0&4F#meh{8FrmbZdJxyv^K+;K{&=2g5!z8^&vt;H5R6`?vI*x;Hd$ zn;MQ%l!I*<{qHx67J)`Re}$ZQC%T8ZwMBl|fV%11JvytS)cAIC*Z+OTks5YMp#jkx zx&7&zDXd>tN_A}g`C#_V=f9QKnB#dhvLlEj30IPnebrUztVDKTT^sS!?6}6_d5mYo z0upF3xIf3)uYl-FXiqw!h3B`$UfXwddy$)rF-0nc%!TP$9{(6gZT4&Bs1RaGglfu5 zgg_{TsB-r2-*Z~@VU7@f$@?5^L<=_W@oVny|L1BfY$mGJoHk7q zai69&j)WRHFx=`Diy?dY0`uxD{cI%@!QNEP??xKC> zwxYvgkGK0Dw+-beYJP>?)LDNWGMjW35Ur0v(4c{LtyI#>hrMo9F(A zy1SCsP}bKMaKt=qp}`tq)@BpZuW#6(HV!OM%_7LCT}H0wbCbsM0U zz;MaEjb~+!BKn`w61uBwL90jp&*VCNB8DG7T6gc2QSIYZ z)|fwY`}&jWL|motG;Pi`gnPa>?dcnm4R2IgLqjqTKX1;szq}U9rHGgq#$T*^@oNY$ zD;R~*4bY(LOvl(MNPF2FCO#LN8>_4FH;d3Nx7wrg&5*WaS*3(6t=YiVMMDdID@#R` zwMteu26f2~nSb>vdLK`zY0uMN@k3b`$r-q^>M8`1&vl=oyk>WaR~fxDP*iz8v$!y% z(h}cF2S-wMxNdaM&d#n{ReXiOFew1-si44SP4ex*hAgnKY+JG@ac$Hhk3Tf_TP7ZY}89(57&rW^Rh!8s{>30vi zLk!v2*l?h#DAOiAV;Bw>UZk}D@5??VN9`Eh)}l~u5W(*9aPQv!@TRJHn;#b`wNW6= z$*bv0I!lxZK5{h1uQ@jj>3|qs^M7-?jOfOkI9tZ(UmRP~&}wii9=h7#lI9Nb7j#hm z=IA=0Rpy_9bE*CP?QL5}$Ia}I(6<{*^9&O0j!-imVDM3d`8$qK z3f|9b@}0veu4iqkRuKqqxJ7|xRTCK1st7H3vE#2T(7i+KF6v6s^>J=6nXxV-#luKm zpvJ;3qm>AZGs!Yna85|rfM~K@XN1^n<)r0=Jk)Cc!~C01QftKRn7b>%D2?BAsceie z;&-s53lbx1c>Y?Kt6TTYh3jO%Zux5L2M_A2hVbhM`oqff_0?`Zxw^D^slQmIPcVAr z3YyJi?--U4mU*YsqB-tkyot)K6UhKIH2GLD4THIlGMX%w%4})Mt~44D5h@07GN8p1 zf6B|VvQ31=dHx$%Ue-5^oP)2Mjw+>zJCk(uc8*o-#=lJ)%vhscbu+!ohm$c8@SsM! z3hBr@|1;cy*3co;(@PLjNNlE0;9Bu^{s^v6Cn$6si|*S;?pb4MLFMj%Kd=_fq)#%@ z&9NGcUh@ zpjs__g(zY?*Wh$%LB$R>mF2-Tj;k*L7W?%TYpVLMBFmc9$U~`)`srS2a5m0I&13IO z^K)}aC@6w{%P7ech)2r&UYCUy_B*Trz8oM2uu~3J1;~&n8po=iR7!91U~4PD-Ua@; z9*g;-%9EywyJcj7DdIsJ8DarROuT=9H6?OR<%B-|AaoN*gE6unZo{Q7TVK z<&BhtHtcSjJ9i>oO9tMZ0Xowc#N0MLYM7Rs6>6CT&iv5m++&y3ca&I}wuSU?^LYJr%P;)2ETW~x@?NTH^S9JTtsB$&!2vu|5jNw z{utoH0G|XfNdQWT?2BW*%iTU!&+P4UV9FetafMwA81HFfaXEfj$5iN_{s zImPDoghmn6k$b7I1?zkr&*WQe{*bA3+whv!pUs{$Ow%a~cG7SV?s@$%1Xl~EjcX1j zWfU}Bj#X`tYiQUi{3@LpP0Q&)0JOYzkKNH2-2b6eZsc%j^j-kG?1pLW1z~KBNn`=- z$&Qix2hp-QQgjuTI)SGLG!DWoIe*S@k*Ww%`nO~tUvedVSC=HTz+9lXt6Ploc-~7% ze#qz{H*zr{g5=7zU#D5atH2?Ux%RvFM@5t%LD-n-6I?}I>$kvr&P*MU>~H<7p~@OP zeAv1=S{BOMRyxCZ*7q9z+z2g2mVJ2D(JvVNarfUmlx9u~-Ni)Tp!c>D+ZKBay6ApY zxk3}>HTtf{V1!`AD_LJik|?L@@nk1A@h&)9qE0?<^o&L?F5+XIS=bZlCaaF0qFyBzEtqjEkKZ|iNH`4U|xBu zq-L=0m=3`kDygbsxQ~xI=uDxB+&7Bp;a2ST*iw*~AYM=^btR=PAjGUzxwn1}hFm(v^ zc7$j-qorT;EGDKy`aaigAXFQDG))6{+OWev>Py_UT+V;d*8vmZ&QcjRWgRQENHpz& zFZ;|T3(}W>e1d}kqCqT8uukN8=~u-dR&bvp*OE3dS*ubn7%)qpM$jmv=XAZ#W1haQ zz~uNJOz_0+*Fa<%QgHR@l+8{Xrjg3tVliR z04XIRpShpsiU6X?<-qE8}N$REtJ7UJv6z`3;0%N+5M6 z9SwbbMW=S1x)I_9HAXh!**Vea*2R;$My20UoIsLTl!OPOoe#sq!+U%5aLR-V>$k?l zcHfR71@h(~X8F^a1rI*hUiUGV$!5&7zj_A=F4k~kG7Rtn^1Nd!x>h7|id17+ZtC4f zBN9ReeoDXKAcML(8rTbo6SB#09h?FB2SLb>P|n`_?y2Vg?K6T0aSqG_9MDYJ7&qaN z>BXWz9A~J5G{?(cA@)T`P!Pz15Cum>cDK2UA!SS`rxm7Ojrt! zH>0Zr20q{Yw{|Q=u8hHrmQi%0P}g-9z$S7LdqV$CuAcNK=(p(svu4zJqV%Hh*(jw2 z@igPWm@Hv=I_>kNhaJ8KFEMF7;z{sPhs{`W@Ug0eF=hl226WLfARvD|&772?yo`b) zthJ+qxSRPAotTYb76oUMs*a>Mt!ZLzz(61Wk->Nz(!Q3xsKyLEL^_dvrKF^snVBI` zif8$caPD&w8Y{uZ+75QJMzjLc#A3*Mm!-qcr%?FOBe=kUZ6N*w_`~p5%xvN#?qCrX z;|_sm>O-%m-@Nlscl0i&jkjMHw+;Ph5{B+~+L>!!4Zhkc6DhfJxe{TB?#)v^++`^^cOLmAXkHvPs5!t!Y977R^-taoJeG= z$p=zGTH`Y3Y68yQbTLMz$va*isylmJL?cVjvF0=!0t8RHBp%&0)z#JCzLBr~8W@na zu=oSW8CKTZ!x157Du@%~>^+u)9Y&wxXj%Ccp5Dekoxhe=*Nt(cV~h+>&FrPO$n(Qb zT;k-B4HErnKc`Nc}pd{wbAg^=Ys{^Q7$7lkzWs?%2j*v8j;(nze#3OO&UYe&Q+<#eCvQya9 z(cKqBdz-_@+`Fx@O-Hfc#;ZYhfYog_+l*~(LEzAsdGEk8GOr`S`4wLqq*pNr2ct1m z@!wZ%lHbb6tw(KsK)}GLr=`?LwwZqy)%Hd0WdGo{S4sXLVqgYPP2~FJ_0s)5Q@|Vu zGah+?D1cH@6uHtRW~|%m)8F(3^MiaS5;1(WZ%t47mpz7EOO`{1-N;(!p>T7nzcNoi zZ!3KS5>!KmWcw=K?USh#3AOpzm;`l`yZWz>W+BFM%rTMMX&*37Ftg#iuvYv4KfTF6 z_qA{t_Yi&L**q$Mx5C1YMMlcHa7F2{;i~!l{T1l?FWtIFEIN~Y(Rh#WV8YEq zrRUgMgK3xN_1;9eVoH9zN)$vcNwZKG{u%ziL-U(9lw$&h&)a^0*8x$y>WV9%X(153 zK1FX3|Do`vP_z^(@06$Jz4zO_k$Ht95~I1)9s00vi72qJ_Swp^OyBe^{qWuLJL07w zZ2qugX=o{JuiaF(Y7VE2V(-U8?B~IBd6LnbnXh#; z%lE3{t<$p$8NH8Rp(lB*udfFq z5P_03Des-4)8K2D88&Lb<3_f8f_YO!|BwXSl~P7W6+qwsNghki5rgG50S?@5c$ZQd zOEzw{)wQ+33q?gmfQG7u54gN0=4WS0UMa|Qnf;dqO;bIGbcf373cFtM`rb3NVRm8D zOhC&)(1zLpcqK~X?XHi9MZtF~=+Pxmu!g4Q4`8~{Mrn(ia?{}B)G%Z=cXx|A2aG+` z6py?sR(s@55Z^cFl{7y!Pge$!ozhE^g%plf!maoCf3&x^*VSDB1VJ)5oqutTniT>TpjKgV zvArBsUSqq$GNqhE3*$gqJ?gHS$UlbNa`+6Kj2#iWclb&C3N^S|_;Lo)60q0xwKWr0 zDFY}E_*)YS%aa3q%l@J~gEXmi_-PvP(8OB3Yy4@sy8b8ZAh2VkYf6W z6HHX#I6(OZUBjej|3l}^egRdy=Z?lJP3*22>voHwGIlWNKXlusZ~G-)?% z7YQyd%+HUo8h}&Mi&`sCwv>FNwM7_>)8;)Fb=}Y{9iT-Uy7hV0nH@jn##QKWzaXoz z2TsRYw>O)TF0HutaGw9L2lMB-uE|;3(*Wih8p0H6sgS=X@6golp=|wi2_H|uKawnX z;eXnfB@P6jZIlK?Z9~5Xwh=%xd2oPx=cp~7LfHn|CaK=w5&X51hfjvAr8u_+II<1j zRDD29>3m|&zeBUPhwt{wT%{KBZPB0}bdEn)8DQHFQQYjd6J1F0j3<|VdhzbT8MEnV zp@SA-VA?dv0wCECMo)4gf7plSF3mPy(xMuGROu&DN8~UXZWP;no6bq4MMr37LYJyS zjdLj`A)#(%m~Ce$2m(|pESoTe5+#wmvMsJo#t-xw&c>UpjnimWfn}Mo}phb>d}8GA9_?kn7w;*WY<~=yY&4Cvpt( zyF@R0sC=Ni`r%*fSzGbzUCRWX)?kR`E#}(PZ`W>nAZ-f=-t6>1+)L}vm_nqHM5*J0 zA@t*g$n@&%&=2B@iHf4n?%adwW8_RL?00k4?+?55v`pwG8MGCD?ryzqaUJadhko|A z4)o*--#(hK`7xG=MzXgp!hXPB$YmAw9wC(*S0<M z6YCIOn|eO;&rKZE z$fYW;W}+63&}sM;A{`h&&-{6ZbLjj(ga4BJhB08Y^HT5&2~kS~5CpJvp94*3j;s@- zXJFL!qqDJdQ;2%5Ae2@Qd)-?E_hAG_En2hQ_*BcUK4q z?tk(p5Q|Y)jpu98QyEC{*)~*N$9#jn`robC_usWQ{=K~&k9YCp;ViLQj}tRD=kGS? zbzka`=QMHyQI0pucjxPxg#48H3>;onEDAtOp?02&G84%mKnIas%XT}hW*~VMO zfXNDH2+)oZ=GSn|r}uhnJ+}mT{nl9{H8^Nf`?T7UKR=87muEX#^gg3_hSqFsiAET} z?=t{(`Kfx;0SG;wx_(sy!xCjYMsDtDmS*WoH0R?Ah1fL9AKsb28S~U|PNa!ta1*uI z4hDQl<>7Z=WxI5u!JMB7N92zq!+! zP#r~W1s8vm;`lz0z0=XqfSvTIxY*6bWure-ia=XKp+N7w3dN<3&2yH?@N@=tS{vmY zac8&svN`u?dTU*r&98l{Mn)CPwCco238CclkpdN2@lD2$1kO`a`3?amYU=02IsqN66{!5ybhA&I`pxds2Kt)x-k6}CNGR>0gqWMMNS7Ec_tdNaHjdgnB zzuYZv08-gH2DC6wqzk_*=%KEp^$6M-uGVz^(-$o4x0@scc+#%C&RW%!<0%8fMz^Vl z=~rk%*;>n(B6p%Oevt)q2D_=+8>Tx*Ji-D3S*_$Z5vd7c|MmztuH50Sk+(fN(2`TL zKDE%Zs!4a|!OgSw?G@#mMcTf17ze~9z<-)O_S3}-p`_K-g@y_z49}t4;GuI?_={PQ zMmR?j{SO$6Zqq_$tL1CNCC{r!_^KAM^=Z;bB8olS9gwAtS^S7$#%xH~UE1OV-_t+k zVZhcuXU5mqwt$UtY@3l29j(Ewux^D^qLciVEn9@xYe3X)g{0;MseMU{NP!ChDoO66 z-1@ON+dJ9=#m=VFwp#9q^p%qG5-_ExSFH0lD@Ys~U6H+iTuA9(&vGrRjRWiHhoFk{ zDOK7zf<}X7t{i!j091_sAX)Dto~LJL&3gj`G_B`NhxvJVc|}B6+=)XIZFSNzu}aJg zb?a*PwlQhAvH31izvYIJ0IdX=!z3L6yHpeCR%N6DQW#NYgI7zA zX%&qF$wD7~yWbxDBsTXeZEn8WO_5@B=6mGN^!sD{qdRotxgPO8{jVoR>XYG=v`d;J zIO6X*P+Zk1{5kY6^TE2Uo%pQAc)onO3GuJCeqPl6X~Um#RFDk$G}eq3>O_Iu9fybDd2aV#oMtoK9&Q0oFbkf6CC5~1s)WbVco{NY zPhE}QL7u()fUS|%$Zqiz0~$auyd7$4N|XgH99v@XUGpHWv=Xdo)_=8YK#RQ>?2ny) z@je&CdY!BR)f!3i!O@YX`j>o$B|XzBZG!xa=e$$7xgFFgmEFz1&$_FTjZ?I9g&A$4 zKe0NTTiksM?tHJIW3f8Xn$5Z^*Lv`XBY?TSv;+H=u2i*jHcbEx&{z#sZPlt`PDNxj z2-6!Ny_REH=ik$PrPJV;G6Yogf7;H_yQe3Nm?=9@@Q~!Brm4DuMj})o{}D#AtHr)= zS>f=9)$(0UHIZbr5Pm{+nTTim#~`fHinyV$O_y)Yui_o{PRFLdp-9ul1<>}NuBW(w zpEx|bqBieK%%XM6)H`EF%vSLx=iq{CS)Y@&;9uTSw$BVEc-V(GfV zOvi_n7MHcJ0?y0s4YWqshlkZG4QknU&hd02?bWk*ir1NhbXLd_%bch&Ri2AjuU&!n zu1A%+y1Gr7C#R>%@@4Adn4KY!_mH8<-Z2p_BkF5OhNeY~Pi-RQ7>Fx&KEZ%VlLmyN zp{`VABMni~n+hfrC(KN?Q9P%V1U(yvREO}TYmpclC_?uicf^%iDlkQ-t^utDz-J_6 ztRjLl>54W}`7EEi4*G6;eu~vAhKC<8Qy`;cPKfnBwPv#}xj<>ub+T=B$C!_nxI!PN z0ELn|=n@{kQ0L?Bfg*1L3E4i#;P2*fuWgVme)nE)ehp?$++q z`IVm=G&GE)!Z7uoapWLxF`d-S?zp+9f{5>WbkIrZ{8$yk$woqXctAM^F!3GK0rS+0 zklm(=(dDUiwT24OcMyCyA0ZQ-1Xph1YIE3V-b;pM^;Q1-9EpWqQke}r%zSSC?Q3o# za=DvDz%hf?-uO63Hy~b{RW;lj4ULnlSO8uQursCH|2cstpV*T?)^c~3ojKWlTv>25 zh<3Kk3UVYmjk~T@%f_(uo!OJdmmJ!ZqaPDXnyqJ+cy98@qGVPm^XlA~*sONH;#dJl+5)mtOBvqH(>lO!f#%+-eL`ED~!Q z8ykSG`TE}fFLQIg9``=tHcEB50d44_F6u6IJ_eie;d$)q%j-0{^jzf z=*sEus)6PoxAavp(<31X39Mx_RpG@^JjkAYI^$ZB5v(*ynrOR@ccGmGsr!1NJM)3a zHQL;P_r2N{Pg5?M@IV`cN4n07<+#(<=~1;un##KqhA;N&J7;m-t-5A zxCCOZzUxW0#=a*fd;4l_z}QuX8Mc2hB;>Nqb5zUzlBgot_dm^-xe~AiwCX=+dL{|$ zIZSMvQ`&KD6gbQtdE%&%{KjxX#YQXURbg4kI8@HYjLLYsU%)$VAOUru2_mzhPD3MZ zh1rJdUWXF9gk4KCT*SgJGGx>l-QLkko&NE5ou5V z{CoWq`g&^dJ=-vcC}89$vJ{g*E$bz0OCp*A~Vzl4~)J zCw(faikYfhUGA8i=jHFT4;ju_=hBJ!;y5zkovL7%L(IL5<&hipB-Bu2D3t2-d9}3K zcwy(9ycvmMV$nt)M`a9%7WLovUIYW-CK9P!&4>MbuQb*O{GYcKs}*R8LKWy#LBPB_ zx-%SK7F}Fg+IG=Gpn1=e`R8%g7wCa-8tS$b76-bYwQIXZt$|&KCT**uvY|GF&evRl zh)o$x8rCv~yr7*~0=gL_+OA$%7RX|)%Rbi!=usAl?!_*ThLn`l4~cK>QUe0W8|ZmW z6bp8x!i`2$q(_pj-mrV4Qtkxra2>$NT3=kulw#5(Oqfmt0%Z|RaG{rx)c=e8kta$w z9|o2D4p*z9PiCOR)*jFHTacOQ$1s{V=ZxM$U?!_CwC;9RJn1CRM*t}egI=I zwXnd+$aTxMy%5!F=~ma^mB$Xo4)z0f3uR?x6Gi9HdIVgCac&FDpq_3%69QhJ_lWm< z?tC4*jvmEpMAq9!Od=q*Fx-t&WuoXR8-$)AEmZu@RE)o?P)o&ca#6M4>Qp!JtsT$b z(I|?_)1Z)(lhf7WI~l-+e@cUQOx$|C52i_%RT4!N8YXBFGmOa?)lbpcJqFSB#LCWr z{zk%On6;i2cF|nHaBdq95f=ltN2l<`LBgireCjJS?A;cbJN0&Ya(l6}X{Hyqx2>ui zGtNE`&A@uf&5hBuBRn7mcs2fDhoK=l?a#nAA?yUCs!Dk#C!mj zZ@bM+FPO1oy|UxlZ-RPFNNR9?z#0wWRwNE16B8+`B`=~To4 zE*>}HX0ywh;CM3-?*RA#4-)1sGX5#R9)ZxdA7MvwMZYl@GQ_~naCcx}08m%EySwi$ z{%|#j>jbI)ifu}&`3@|DT9HE8d~|679IGnSK01y*Q|Y2RrrBH&$SZ{?rM*t|FrYx4 zgbN*;7JrI+PK-#$8u*t2NeuvI3B7&$J>ZV{ri*sU z2WeOR6kFZccm<*@)Z%_%R3LhSN~yYqffAc{2j^Imf6@2M=|y^;sq`(u0Y)&QJTBXV z)ek7R!>?id2H`sJ4J=aWJZ0Qx$H(w=a7>;HC3{ltKMasVu2eUx97!GWVo5%&tQaNc zKU;!fww2bBj;+IN`|GSukN60sYrdQa8~vO1vY)K6#Fv1zjxccl1TRAz4ZXWN&}Ve3 zHA*}366MaM^Fp>hpj)D0oKU}QQ=5l8?gLA+18y=LDm(CPDw7kGVm?48=)GHY4&pc@ z@h7jO-R-%WZEkLUdkYyLgk&ly`JOi$Gfeb+3*wVRw*sYwyuJ7vZIDy{5vMaJ`m_f# zdePabgS7VKnyw@1_K1Tcf>jD75A$s&&7YwaRC0BD`U@dn=`rv>n(UvD<32hfz+o#J z-_ui)KoG{Ib7j5g_0@C|*uw_b*Vh66fk$weoti`1NBG7EK=p46Qm+fe{~n7~KYU^_ zpQHb}oG;+)&Lclg;ced{CK~+s zLjsuJA|Cx1`+kaiKV(X?mGh)#d`58lZDd?p(`xJr<5sIN>BeI%F2r6Tdy{CQD{AkX zjNoIs2d}O0v$8kcj4aJBUBpSWx%V0YSKE?&2)=?{WHC{N%cMN*<||o0U{I&ZCpv+V z|1Ny^&LP9-^gOT1)%&oMAXnnCj6Z212)>N2M;;!M>-`loz0 zTE#90GmoWi0uhs#ppw}-#o?&p38sZFQ3I(>0?Cczw3e!vI*G5G9Gf_gE;Q~A(Zwg2 zpazzj6lH|~Q6B9Dqi!tRPlNLc(BS_h&zF)m+<3@pOH0GWy7hdgCs|)arN<>wII711 zl%0|ppZbBT=Mtf9F1=AbGvfn5svCQ*Re%N9yyjD@gS>CP7ZJEyzJLI!b|tD@WgVtJ zwC`iAgucx%qMDKP-PSB(+JAz5mW&{J|Fy8F5=_DSFu@T5_RKdtII8dL1jpNoYP^{$;LYiy)k(K+OCDzb% zxd(}Vfh&j#$pf)hE!zfn5=umoU~{Boi39*c{Ix-Xj1Nmw5h~+hT-L&K0}&Z=K$2qU z6}X>J<rSdl41Zr5zHcWt&WSes}$8s?QAHqFb7Nzg6C&|u%mdkEV-u71 z<64U$w`~nQSz3b~KF=lA{L{gpN+xBb5;4_0q&=M--OS0}%zwBE)h3^~#)uH)*^ugb zSnOW}X*$b4l-RV?F^wvocbl+?aI{Z;DPF%AT63yj!@$_4p!XLhB}dQ&`+$z-Rl%zI zAd(e-1=P>%wqoW@ID@=E*oEZDl($DCAd(Bo4iUDFKGdz4uQv9nVl+eYv)Ni@ukAWX z!c4*cadZxTb-!&K&*sV2$u?H4PFS{=v1PYq8_UKzxus>>T<#Z_wT$I@zI*vmw)y8gA^UsC__dK+eBis$+fN0Ts6{ zP3~s4Q+^KvYBxWNUGb?cy4oF6p7+#cHnvCGnw_LFbAjni$cVqKW`>1o8Yc4e(?hik0EtIyZW7KlCa!v zP3mpsv5X*i6t2C;3TT?p%R7cDlYKrM7vmD|@r+l72{I->K3ibin}v`H?jS-; zl5rIMOXUdm1T--iIU_Eb1U}Ps?)-Q&I7|}Mybg*QjC5eV4%S~8X@oR+RoRpDlo2IA zWXf0Gl&i5Ff+G``t|(*kCop9tMns(bWHm&04C%7<4<1r)#>xQip6mh9j+Lj!dyO@p z&~4P`jlQ$M z0FQW3i5h9Wj-Y*~LsnMcJ6n+a zEjKT(>t8bX(pZic9&D?uQ=qfWZux|}nPGGOFGsjPB3Fl?2GZ+i$NOi&ehJLuE}odm zBGiIO3kEY^|4`pZ(j2SB*GepfWToFdLoQbocdQ8=tEAoivK+Av7RTTrxxdE8wb*Lu z1TV2=zkVw?W~wqv&*#^an%D?TXrw%%5pYfuA)rT%A&(E;EcT zF3cWxP|O7I#^Rz+W}vvTmj>cYdbB83n=;T;J~sxBjeM*gD7c`eT$z|9r}48=eC9ET&)ps+PBfuyiq>R*rZmVk_9zaY$N|Om z%!^1){&Lp`)U;CkVIZ?t(%eeIHfg#SZVN;NBJ_L%1{EkKN7<%S$>FvGy3O_&psoNC zYEy0g)jRo)>O80EG*QY9k3VUn?%>m-EmGeO1cgmNyQp-4Pw#SO{IHmN)uJfKLibu7TMDGN!QAk(|B@YQ=bvP3Qc_pgz(~`5eX15QNF9$jo6Lx}l z3TzFMaQZQDc$Pw#b)?ht?hnI*IDpv(QxfbamsNh@E*1Q=b zyZBj&Skc%(nH+KKDJ9J9uI5fmUJI?41?!Jptsy=4CPwwWjgY+yXN@Tf&WI3rR8O29lrWK?hw`|*oAPtr=@_G zrMuq&RDR=CHDzDH`ox%*sIu_3d6}6HgPi`C0N#b%qBpEtexFvLTq7|3z2^nY&Hsl? zVOIoBe5_73B&fW)ZaN_<0jRbTd8EKTogvv=9g1yuv+p(bSbZ7wq1mzIKZc~P+u&(l z85+E^D@zm~9eRXNPk8;7zg2&t8PtW(z63oD%8iE^5k#l+z=bBEX(&;%T2`(U?Qt5Q z6zTwpw}6Me^dsrSBV@5x)>f6ry zR|{)v6vo!@IA$|++Yk>X#9!i1C%WB_+J^<+Z{=1qGWyg=W}I9j-x{sjen?lT@vASV zMtH`?qepP)H@AY1|EB5+jLA1x+uK)mb{d)722O)aE*Fpcj0>G)lkkruJ#Y`3xNcq{ z)H8K4pT$0{a3C(2TN6bunM6yuMm3uS2SPq<{V5|by#REC*!Xy<1B90ub@{ohAIz1K z&x5(Ziy^_4Q6(qv9?l&w=MXBAqFKSK7Z(@Bis|`{{ujN-Epe@%fg}r6>K>^CgX#06 zlfN4>3+4Fzh->@fGSOE;jQPj;xo^}?Vpg=18$gW-tev{E=GKr zm*;=qtOk=Y2My9bpsww<+dj^voyOT(n##2I1m4))8-)BsiirN{3aV7c2ZAxwGXng zT?vGl1D{2bg2$3Mg%z!&gq|H&n@&0BAjNVc<~D@4;o4y1*af z#@}=Xrr2ppnl;l`$x2CnD>#!!drNV(BJw3;G6Jl#^}Y05nEgn~|JuKDHcZ#}o!b3F zw5-2gxJTMIjC7wlA81LsvBcrx*b)=XtTY!Tqt6w4(j=yvuXeO~$efG`4{do41dT8D3LkqvR?O%r8-TFZYFkotWM{Ja{TA zm7hh4+yOvFyZ~JvYMAcV+nSw>bHl%f`W1R}AW;%p;yb4S18%dtvyH;10Y!_wv=)YY z_fba0AHf=Bi4f*s0Ys)TT2d#)wWT&{_s<_7ok!Z!c-l&&#J6fe&NSoPl8CXXq28!Z zKMM{5pN5^f|}wx$l^!J7h2`-q2DlA!gqwOEe5aDT#GIjU+}#`%yB7fnD_ zvaz9os|Zz!&I>2K{Z>XsR7yv$;Ic}s{&Ctz+;xG6e{lA?OJD1U%=Yw0g|fm28Rw+Q zrNRQ75a3k>_$&2UDMEv*T01baLbUVV(1k&s(CSg+^|OykxD(Tdr3Y!ly48jcHE)lc zdf|XcfU`*^VDEXc32V^XX-=rc6L8r(g1(M5p!a;% z9Q!m2hsYR&X{?R02v0_(ZI~zdq$*JH`$O%fgO=qYBLnKr+Tcs<8U_x4da~c>(B~w* zqUG99Q3PfE{02RzVIMQDzgl@_olDOjVZf_<;rC+%gyWj-R}p-56p59E1pipattJxV zT+QLnWK=c1kvDgDQuY5-j;`A(CrM7kpXmB5`Fm%v1qI9lijBCB7?M*yqj6(A3Q-q2 zHi^pUlH8nh6Pl>KWK1W(|KVn1l4?0v^YZxnVr|Uo$f&fH;QUZ1y7HevMB?X1beE`QO2@mQYd*lq@Eb5~fm{d-OWoh!mfw67 z6qn1tU9U{Q%5-B{Qsj^K7(o(YlKrMoRSbdq*zbB$(B)dN>oTrC0^7}mFItlYU4-#% z);%!+SvP3Jfj`qe z3x}!|1WFx>ZVRBm4oTx?luA{7_|MM$P>~cxrLh&xt<||q*|#iz`aLuZVqy6MT_2sn zhOTu4M;cYJy|a_zi_wPFFjyqJ}l_9n0l3V1VfNdTvQCgq#*8|eJua`o_BgfEMHG1cqQI2T`^*ry2 z)g^>g2nchM7Hz6%g{Is_?o-Uj8BIrvThuuxfLGCEgIm+rxic)s-JoGzcN;$MA7|qK z6|1;#1X5r?qodkk)2IJhz!N9R<3LBZ6}M&+dHB%Hx*oXMT0AdlsWB~*D!3bdg$Mck zBmzzzS&uuQC5?f)DIHln%4zjIkkeU8T+v7nwv*X^)+hUEW<)Hj&DFfOZBj$h$|I74 z62~<``p`yV$VW|-wvzilc6Z8>s8X5x1*Qcw^L|R`etxz}94g-=AL9N2C5QKyS zp@NTt>OWjEx`he9=pDj0pWOr|B+^&o>Vra~6p(`jCEmfc55oK@E-=Q9Mp8v_ec&AJ z`0;l#OH1kt{D12xHN|Xz=n52>%hY~7?lb!tjeEj`#_BDu`RArZ8 z?NdTb#yv#C!N_PGxJ>^Ee1N2WKK%Zbn<+c_D#;8s4Sd#HPZjWd*?!)&IzJz(x|wdq z7q)VeYE-9IzIa@g$3EYjDmvn2cs1p^u-P(TPS^)nB>D$p)Rva=1DSX%E4d+p&UPBs z(sc!Wt5cqOu|(e&F*ou+pJ)R z`QNAo@V}0K1zCC%3+9J9QA_``a!;KCo;EstaGG*<6JBITlP*Ez*qE4|FMn5omaCyU zjiY>L-IvkFP*f*+*L9xeO8AG58pRWC3!(JV`e`w`6a?wfU(E%WP?c*<7E%<%Xt7)# zin;#Py#%=NaI9Sj=67LAeP)^VBq z(~$0mNhi2y7!V#wbiaO5T~s;&)1LTk3(0}m33LI78m4m?nF|RGRvP~f4apXCqI~b) zQCf<8rM8NtXn0l{M!lO5#j(>Ue&ZrP$iso5EK!|S5hLqcdg5^INK082BJMISz{Ct< zza_GST?qyQ6`lP@t@V|G^!H=va!cT!(UT@UtG_Z&?s&D;>y8zj`D~l2JlCD(z=G- z{(hF0p*@A=4CQEH`_x_ev^)suqq(kdA*Xw*7g|8z4^ zbffVeYerv7$rcCVWS-`!E=doTd)Ls}X|OKB$TGrxM(*!jX~eB5xiRYF@?1U zn;#ZG3`OY__fY#26Y31_i5Z1>mL-Xy3lgNVvC(61EVaO4?8TVxxOORKPq)foVR5m^ z`_?wu%P}?!nbA=4r3_@Le(K$Sj7JojoSYo1nXZUnCjVJFFWvz8QZdbD=P{zv3{z6Q z!JCkLd`h#;Q(|P`1d+gwF7TThqSJZ8c;fjMDL@=xO0T-!an z6RKM|jE`fZDWPkX*jL0l%gz`H-4^|IqpPH^#M3xhA?`XO5T!APDf8%ggvk&5EmDm_ z7DWU}1#qBPJ02O?Bxv?!-u;Kv{OTnGipJRg!07A<0pw(4g_;1;A7HABD5*@h`*77hg?a0)` zmUYdpnqN!t6zUMTt=4(>lo5QEw=MejvZJu_wJ%u2YlHvM&h0unUbr*5WO!!6#M?2$ zKNqW^bJX?aJ6iB`JGKRoFMHu7MOt{_7vHrr3s&N#lodlcdkiRkWncr*LMqXo&PwEGA(`lesK54L&yAzpHHaG!!Cz=Q&d0+%`7 z@5Mcr$9&E#5fo1>%e_P)M`1$@eZ=!@0)r@+ew=>|138x~AjHl@)@qg}BUKG5uKD)u8*t^46A~GWjqqz$W~-A2(E)|#7!iZ^B+Fzz zdBX3Q_kK4sA0}QI=xAB9lg(x3wNpUf_nE;&<+9tnAO9l@+xALHnnbNN@YWV$C*na- zMhRt5L~o~Etxv)~nIhRI;Z5`@!;8hkLGW2KV8)XS9uM7C4U=X~N9makPDnI$Om;0V zHmnc%1$=l8|NVr$HGA*267C+=os>!BcbokEd3UYVt&*map>v9#`3t9;2O_770-8A$ ziZ38D3c-$0Z*Z2#K=Kc%g3sW)e#!~j=OtN^fs(T~iK0<4Ai5oTA=Yqx-R9M1qs<9v z>2D6Sci{Mzz0}J#S!M7lX$I|ry1W;y3MuAbPGPDguEw-jPqtZ}03V;{(~{WH%b&eS z*SgKhfEim#Jrh2;{4had7Z#v?1di66)y|K^OR^5Z57A+N>sG1}lm43}BxS~yw5RdlLxFf>MUrd$5Lg~Z zt4$ZS*1;bmuuN1kJnopm-5ZlhX+{p4X*&nU6{sgL0^?wCLJ%8-qJ@4B^m&A{h$PMb zw!$P=%!g3h+`BQ%K3jTbK#lt+yD4e_&Ef?Ll14L$!aAglegZBZ4*hWTrV<@I; za@K$@84rO5A^t9xTVAeMCR&b{9o55ucC1Kg&SFzim1|U|+Aym2b6ZNnUu*K#1AZj7 z8w2=qlNj}bABfh2YC(Xk0^4WsQ4(4Qgr4X=d<#Si;wZ6)qa0!DEgcbOzHx?k7UJc- zCJMMD`bQP%2724cVe~8^Tn5NT5$O_3o{a3kOq4L`c*`}TIgE+z0y4*;4jnXSM${fQ zqgj0)Uk4|94tUJ~prpVeRT(QV%(C#VDe188uyl>^h@)EBsb_KQ>``3y5ON%5jQesb>Od z1wY_Vjz|Du-cwivE8_iVRH%DD(hqR;g1t@wJjq7`0mn6zsWDN5hCAw)XmoQp}O!+{n{jRab50enL*%&UO?h{+26c48;Tp;YHY15_BuxoNl zOHb==8P66$rk1EjXZj5YITZa8K;0#3 zJmEVpMlp1wRohStk}QiyAGAZEcPBIF+9l=3#WB~)|7MFr7+n~Bt*hG>)y8B3CywbV zxp&jWjjI}qD#hnITSs}swhyZ%E(aC!`5Rf%m2#C3ih}abVLyymG70bl?pcA;O|O9bv|T?Z78(SHi#iFT|THyNh_C}9)~@weD(JDI3a|*Q)Nx2o?gzgHqb|7q^UbIHI+ZL+3RGEwW`SR7VYsRk2V_v{ zUM4>PTJw!k3@tU(PkWFdLWFA+#9eqzgVY+9r(_y#X*#03+10U2a_++>pRL(FhipKx zXG^2v0eZLD1l$5p-9Zrw6f>}(x}BW_oiGgq8tJ?{~iSkMU$U)Z$Hbhzvk zUy-mvs!EYC&}t9g3-+cNh>l499055gMyILz>=9^yi!T3V;NJF3Ug__!cV|CuNX#h5 zsJAD2>|;g0`@teFus*@@B4to^Js!e^?*fDHx?u7`AdVKcCe-13dFpzL%iMsNuZ4|` zEhX(d+lPrd`}f<|{PppcgQA=3C$o^T(pu|;ycpsdTzt-cvxJ+LE)j1`+@Hz zLB=i4uzmw!2d>xVI_xRhBb-pe*Y)2oMnL}!0Q6gEMUe(2Qz8JRB{$eXw1W1u_(vKsXp6q&o#A}89=%Xfd%R!)SqumsG7T~B34I%$2iI3%*4Ly&wki7zD=G(>uPZDJ$&74S zjLOm*xtP;QUQ)v-FX^MOf{{Q5#E@5jX#HYKplQf?|$9` zfPa8TNpdP*P16>~JoT-Ais&tFVpDO^jhm=9@6ull{wFd5fY2^p| zNe>_)VR!Rksn-Ig71U9lFCfAWVJKBK9jUVWY-ca27Dn~aStgAsb&B{E0D{K1$SRM% zpmKjosWKq)C}m=Sr(wk;2OuD(7^W01?I=TG(`Z+=bPV|yp^paDJa+l=M$$QKvZ|6- zIqw6ygxMi`Rx)&2Y7HY+KllSN7nw!KUts7TfVdo>yTHe9dK(r*+GdB5tP`--kp~@$ zb){uU^Gc!t-5hVgYBECrP~+oih1yOfayF>naNVJ`T(E$I6^`|xojSQST#fvbd{?W7 zrZ^m)3C54+tA}hGa{j!N_~D%3Ay_5H1f5-NdWn(cJ|M|rs~i7ckd^kzYW;jZa9#}! z51X|bFc-&#i0I<73$cRMHzwN@5QXsHT?z1#tQe!_AC`Cpjz+W!Lhz$&pAGd6iV*#Z+~5^~S~qfIlQGp|&o_ z2caw!c%*#~ISH)we0h1H0yuL3Ev6w@r@~soSOk@#<<7Il;I=#s3JpsLRT`OZ8*nJ> zhi*!5Xu^g5T!AL`#S#K{rj~R;26<|uWI-A?3viVMGUfMY;kV*!l+ZG(AVHmel9xVq zu&pEjfX~01JU|E?KOB%zKHY><&X8`Yk%0^vWf9fzFvwWk?k~2WP#SB;L-L{a*`)%+ zT=n)3b*7s$OkJ=Oh+_ioMmOx?8U<2_e;ciH4=GsdCfPkHLvPO7&x5O+3$wwux{7OF zzC)Y`Ug%}U$n8E-t;(!IGDRuGjtn2W@S}B-HQ+)`3y2j*VH^m8)C$r*rD`8y6&t{T zVS?Og3F;7Lb?#b`hf2Q@<`lyOcS6#pg8&A)l%d3jV3KgvLb%O$E7{WTX zf<=3PQn;x$jUOn1bGf^_>uX^8VP?jh2Qz_{F@M^8q-9GV(7iC8fV=hR-Y;{uOnolN zelT|NRGn)8=LCi(orrOHo98SC+I;3}a`^HQ6m%v}bR6|dlC(U^qx+py3#53qfLJ=} zPVY$U56Kd(xtgo-2-=K{qpS4tq*@)|wyS}xBby2wy*f-FpPIw8DQqN~am0sn`OMNG zFCn!ESHk1AP&GPF4#%7g zLQ|-r5P~x{0n~n5))|pD6aw*$cswD|ps6`4UEJwq%=U zMA;fR&o&ttCU7QtDXxIk9s;G@N%#aicfPmVnqA({=YAsp>LppFDL>mnqE2z`ti!yr zN`oQCH!CAR^Q)A^KClKb(w_buGeCwiV_t2SE4SH{WU2r-p>--m^O**7M*UX|SPtA8# zPCh2)ogKl7F>JF3_>M}fzR&lUf*YRjEDdZcJkhS;-{p>l``M(SI*mpqfvXz20$B>e zM`(@j#?cxb6Q)gFO9+ng@ba0S)9^zWD7onnNl8=4YsF)a zGdNIoM(;*vmUTXj*?Xe$k{8y|c2rh)Q<|LOkjtp-h`#)7F*m-vydGeT8&LovHHGJrfvuSFaOsFYAs51*eI9Tp_A+ivA^lQ*yqKr^C#Ss(efn zC}&l`Ct`Ck{=h0Ioc?Lc%fJy6RgOvk+vq=~DVw^+-2z*m_61BaH_k9|hdKhO?zflo z;o;%m{!b1gZ@}aN=O4B7G&Ekf(`|`sM`(fUVD9XE0fg%V0;QUH$31Gb45@Zvo~tW8 zJ9!Ec3F{WHQmB5~ZS#h*wSvt6KLdEUvJFq5m9lB*M1Di|lp@{$mM|>WvFm21uFHLG z_{v6?Ua5LLDe>-b{{XNOfn3@9$uaZr(|R57J>`plsOa<5yYn1TucYymTZaeS7UKW3 zF0~yU*l`r(NU*6P)TmO-?Czt8km}%O9=MO*S%QJ@-s7`R<|#10TwxT=Q}d1O4Q{iP zxr)2X6aAS?h{(?dJOEw5F5|O@5WYzjBa4H>UQur_!m&|8iKPWy2|m&cLyC^Sl-q@9 zHP>d{AElc0y7>9M0G@~2t=?2T?3^cas*wXu_&|O|ZusMfLuzHqmg8zH5%*vPvL)Ls zNku!I7l+|^g1(6G$1n?LN2X#W7MchAig*6;b~Nfdlk>m)|MHd*6qd)f1M(tenJAg~IA zPVd$hc_zoV8nhZj`O?;S2eWLG*okQS_rX`dsQGW$!+!sq@DB6ONIWCG#N-e4kNWA#( zCDZs@wq?6p6ea7ZuQF-RV6G)Kb}PREP0V$jh{%mA9zXuASkf1nt}Cr9`di?~On(?P zv#aWyQ4d_suO_d1CLD$>J+DPS!o_!=5zfnL;pm#R(L=95rwB7l30{x}+-|$Y%H>wK z&r|8?xZOURJ+Kmqzp1wyDcNqB&?C-H5+=knSX^naEobKE@uidH53=ZCtNjO7&o4KY zN-SuML83 z?ez|TB9BkaUlMpbD@#aSez5>?oMC*>=s0X;(NtL3cYR4rtFZ(+U20_4EtYz&%IZHi zJAsV#=Mh?*5qyR)J~LD?hyfYjgSEQ%aCAk|R_Whys|a5=fWW^5jf(zNU3=1*v>4`* zy+J=_DOuHlkc*}qo_Y^--V__6)8S&H;{=vJxyF#cTCvGrUR_HgPW~YXNfLPl@O=`S z*uV~^o#A-9Jvr}w9wsGmT4t~K`7u8S?MD+{+&HPKbl1P%K2sxIPs}JX!-p~fC9Qc3 z$rSmv7!zv%&;@-vSY11g`)6!83kCxe4zih2|VOtvP zpyz)@vJ;a(ly;jHjuC9SiH8Z&4rUJXeKIpxJ@mhFgs%&!UJeDjAL6Wb$&krh5bG*Q z+xm?P^Oowb)AZ%Yl->IQ*X+5ijF~7DaC+$I=y1Z#!x?NMC6bEs*)7pUh@u5wepF7f z)^#>MpnjU!r62JoG0M#wAoJOxD8*4(%YL}s+S(HQ?TdGH@yF!t5eUAnk*KqieINVs z_s7s7=SS4Rx)B_7HZx8Mi-4EgB^uP;diM-D4f1~C88rnKPI^1meR%?xLy>otsNOR& znGaZ526lFK{zmgD=n&uh)>bc9*Q6Pj6i}VGL6MFmeX}7L1+S5VL0;3kU!XsmCVBaj zqU7u-zEXyA8#M;K=@KE`$?U(w-Xs8U5c6{sw~dTGcJ3H79-5NGI1?83Sr=Ez0SWkz z)Gx+9r;T%^s)Y~0V?IsE%P}BaWO;YvjQ90NPW&!g8lf$L7^_@6DjzC0@#xVmyXwpJ z`!_dzY#1jPnot_tkq4?}kWsq`?QBVK7mob6gNBafJI`_dbgu?AyVRd92}fPMWn#aT ze^CC*aU`pX2f3gue6OE1SGE0?V2PeW`1{$4?Pg4GLgh+l8z%ZO;O#Cz3cV8udcS<< z;dJU`neR+9Hz9$O`l_?|wkZ9TW}Kz0m?~(vGmW?+4vx7ID0cy>N(#X-z(D}kSq%f; z=fMGqeqSoHMB#|oa^*OgNm{pD2a}O_Qg09~t}V?a=XKgu$~z##553Dc<3xILvyll1 zcrS$87VK8aaQ#P5>b8t-nkj6Jshml&1JEb`{l=gaa!4++`kq!xpP5xD_>a~ov?LR#Q*idN5V7b5nM+%xu;N`~ zbS|xosG`U`TL4U%AAEuwxqot1qDO?ZZ{Fg)*F)JYlK3Er9*SA?0ATw|P{6$Rz8oY1 zR)yT~?P#)mI4m><+opGD>(u670~DA`r?gLh??p56G*oHQMR6Qn8SQG8mct2cXT#T`?(}33qOCi#l}v7_0$@di2q2hNY59|&^^#Z) zvQChb2=^3%_W1A1zg;qkkTspPH`V$`t8Q~C=<~$lDz6~9hpeAf{gbC7k~SQkxIZD( zYmr007=!~^*=X~yb9bM&e2SCPFv3D20DaBVV)eB0Q{SK)(NCmbuz=$shXJoCC6km4 zMsP&ek_iyFP+G&Pyp}r9Km~@FR7>_Qw)(loGD#z@=T)u9euuAAFxI74T60#9Ue$>e zU!iB4Q$=}pIQs%nsej-3C{DJkb^U_^J+(Au;5|^D0r(H3B>@WPUeGB@_PN6|g*8!%QdG#(u(E6yYn@Rq z_yLj40=@PsDU7_m2XF`Fj+?;CIpP7(3`zRzyu5Hz1T1Kr1l#*m`irIs6$ks4-$tZL z26$hi42BvZdw#5zUXFV7hdDoCek-CQP`{2XQY^Y_wf^u#_(XUR*9|QZW3WrPDfr?N zJmk~!k>%UpIYu57EA3UH5Rjr2}G!6hrLJ$S|Tnxfwe~e zE8V^oRdJKHyf&4~|uxVK`)$6tjs(5jd-z<@WotnlPut=k^z1`;s43!U~X?bn5 zTMCpasX?aUp+|V(+a@i30H2m~)SE#6&Y8cypG9tqlC0{R3R{8H<8J(J5&TuwsCAQ@ z_wFl}!6P6i&IpT%iD6JT2n4G}7jnq-WEo6x*e06mqw%%U-^T3n(0_oh7-Z@O4EcXv z^lNK^qqP;g0?V>uykV_qfhxhdo2+WA^k@$Xg!kRMcR)+daQT1*0V>}a1}Fg`>Y?5V zFK*~Ws1~WtXw4Ziy#6n)E<;=$pKNW>im>@})zJ+Pb1fg#_b`EYP<0NzY@DIKq@*lU0LB3M+|cS8?$k5GN6#($z0^#VPn#Dg~_kq~|pRngA|!RTXUP;O0_2`poYnNCCzDoau-2qw(jLGrHPy+2Y&a0b$>Yi-pN?24VU2-} z4FGJ&)qH_-;_hTwsyGa0b7RN-!JxjakqbR}8g@#xztXjnW51t`A@4bz}4TMvXWG^9l-6jvw1TQp;tZE6JdNFeBuTh+Jpt+ z!ruZxpr0iYW5i(RG2pDWUu#L^Pv^Gi&eZBZ;Nt9ZtEBZeNS-VNMT>Y)prZ4TH-nr0=41;T8H?fXC%$5MtQ7J%SSU}UqxVZFevu;YCk&xLv1!O~Vn+*Q`q%y; zu=M)Sx~9U$ZA`FJs@8dLfJwmpMl3LxAhJx5vO#*Mze6lQNZM&KPvyt5E}Pztso`&9Dt|;v~xhySYBIGIz{ZRWaNJ!oxDPavs$RTug-{u`*H%j zxbl}Uu&@Ac5U8X95Vi9w7e+CiPxMN1; zX8a4{Cu0sbp_zOq0JI93AOd+J9%zK3e35_}3gPO@p>{L!{ho5dVGy_-A4K5-!05G5 zo2X*-tn&u)EJXrX6g9gcr)GHw=H_($Z6mlfv`bTSiinMTs--T8nW(i$t+uH9L$uu= z!)lSSxO2F`mPrY1On1cj4^vw=96UEFoBQYdON`Zi0$S+m5n=A*;$42W{3>PsiYNqMjtrF!1@{7%wPbTqJ=szvEM_I(G*h>pe9$c#%x;2nY^v4gM z@C333$3T~DkawB|aF2-XDf<$b_iU1mI(odd%ktj%VYjE?Ar68ce~rb~cb(qZNLGP2 zjqY+sY^#>XBnL85Ok6zLc$a7kWEMg#DV5RZ-(Mh_Vq1)sZve-Y>$zh%B7AZhtx=s@ z=%A^Pe>UO82UCoT5GJHuSy@?NFzgY`6ju~cW-MFE3TqfYPChm-Ip&
    u%;ww~Hk z0r5!7f@yG2&h#%ELOHZyXpY;g#tHSriER#fsdK?`=8`h4TlE*8#HyP6mfIo3&idau zyfoergU z^qga6^!bgtnpUHU2^}%#qmt&>z<@@W#A3C82dVv9S?iu9bds;zRkuTK<&+4#BV_l% zl{ToTiBPVAY5UbCOo74e3Uvxi1UB+m@*U*^e$8dKK}H8})(*8@s5;90cI_8&J`j_C zljYXA2s=U8IBy%YxkDBmfcB3O+Gj&8Xj7h_cdsEPml|(80iWmh!$`d;OQU> zwjk?=giHK(SC1BLaGjx7X=xL$=D?C4Fn{|g9!Og;Oh63!K&v_`i%$c&(a~W2Z=-~S zUS{x`W-E)_GwN$$N+En2q%xm}Inz8@)|c_hG|5Vv;8&@73Xwbt$jP%ql;s+a@Oc2m zy;03&LW29~`1mp4)JWb9Ul-(GoEefh0|(KtgGT_N(3sixAbq&sHaQES^R0Etr^Y~v zk&&ZrXduim3(kvuG&Hf?^U+=hts8aDe?2VQu=6t6%ku(COrK7rc*SL&V~K=q0#7q*r|G&KFcZjCPvyFQu+O#)V&u=yRbD7!~7%u%Reko zUQ1?(DCIC}11m7odbD7US1%>?8CKgIS~R*7zTd1kQbwRCZBafH=~qR?G5b7@vD1hS z;u-hS{OW?tdZ{#m;KaRRCd#yb-e6f+5e@+jeNF>Ap}}K{qy=#(_XY70pDl8uCOEsg zY7A@wKi;esMrDSrR94*q2H(3gisR|9&;p4+A34*-Yog0}!K7d3a&I~``^n{GwdZ!F z^5RCE9gw7tM(bYv_hZH?9W0_;R?j!(W=c;5zyIy69}a~rEs)+{%Sz6X%E&Xzzs})K zv^BkSLbQecmk~d+CT8Pb5Pmpjt`hg)i=OLn9+28yk6*7Q1d5xE@i@X17^g zVxKdOIi$Xd3?ib05At2xjmA3OYh^ZNf8RJl7fjz0_pq_~q&L&j&J!T$*)&NMf^ozC zpX@8Wr&?Q5yEB3=u3#*0I2}FHiz)Kh-|6Y;<6~Pl0`+`)Fn+N+z>qMgLK{}o-qhN{ zP&~4xzX~t%%4l?64e-D<81OKkr4EbD-1$?#vd6OE;V`wSt8%%R?pv#G!b~U*U;kXt zC*}#Yx~27z&tR^21O^EE!EcQ$PWJ$Wn-m<`N`ve`*^3sNB6`%a?&COfAJXq^;| z-QJaN`b~72HS_TYhkBd8$@LhDbNZGLzff-c?tES<`2rof@OE_qnHAQ=XwA69kS?zI zAZD{h`C6A>yNu6&eY-Con;oNEP`H^eLbLIEMR?S{BFA>HO;?&YUW@5{^kd_C)8&a* zaZGM&wNMLxA|IKBG3wXRryH}a-?)0Oud9EGb0|Y#X}YVd1Cqdq#*hvNcF-OIQ}7&` z48AoDDu6HOsH>wL{@L0ZH5gr3e1{!FM#Gj2Sr=YvNz(#bQ!z7=&~h4L7n?g&UzU@~ z?xj%Q*SO=uw_^>V+G~&!6Z0L}$mI_QrUPEX>&={M)oTA@S}nj1d)wTUz+eIUN|^OS z+lrQoZAdJGVS6Q-gj?CexYZ^JWXKKw@XAfnVHchlw2+_6gy?J&b?vE?H9meW!93{J zM;Vi#gIN}qt)_5wm+=c>vpMQ&Obg{mi+Fl^vK~MJjk!Q#d!#MxB=k(CKIW=BawZG( zFhT{aYI|AO|Kj9px#!XDLblc4a=ZO(v-q!e^qq41-Wl9|45L0wGCQ!J#m4m#l~Iq$ z2nHlQbaZs~_0F_pXBU^msAp3m^kzHxyjtUQVY`O1ia+QL25m#ZoVFMP{RDAa&~ELe zpv$nlSVyel4bUot=LK`TP@n9bNFXO6) zmv>ot-7D$mfA8$r*V9)R!@Kq54P>Q}lc0`m_-X-ShrMY0h+jWyF2C&MlKl9EB+m>Qk zlOVUZ{#L&^5!8|CWa4aC+wXokv_3|BeShaF+ za(*`-Sw5O(0HKwXmI5R%#-T0Xy9o~u-%OG;p&R(EMxP%RF5D^>g-2n7C8NsN(iM75 zfnxGdS^m5cQ2uZ=(HIEx%o1cDL;DEqG+FEPX?AE|7YCsMJGjIF*f-GS0J?vuFShcf zBm&mXE(5`n3`&JGORPXIQ?Tbw6W=v5@p|zf89Qjcpipp=6~8hL{((#%6lcl4D8tg);-}_RWW1EtAxVI zG3n{+m+MpGq=%#XBUc=7fpY3;!@S&~aRttk6BCgXp1__^IFds&Mrh2f#$f@lqc5y4 zFuViEXj#|oCbe;{U#rg5Rmhk}$>v`R`i^H}i6-OWl}je7Nr(Bn4*{>$0J5FFF!Vu6 zqf4;63yuj2gYCioeu;G{3c=XX)5TUu$6?a`vS%XqOV%;N328+a^Gxx`*gC)Pa!G)| zTwP)|q6~gs1_0cRAp}Sz214*g5eaJ4EI~PAfX{OZNC+YkLK?h7rkGj!D((esNkN@4 z`O0KB!4M+E7h(*Amx}^xJ)r0Ww4Pc4&J!covbL@0uB@!A6nzp)>cGFhU&B6Mn0AV7R3Kr(d481d z@sDTm>drmA@BKG54x}$!rm{?hxG2Numuf{Z41fE2fZK`xYsLUkLA@MPQ{8_N7gkZm z$6BS{O7C_Ze$nffJpSp0YXx<};=%JJO@FtW%psk*^#JX@teE6c_JrX&{xlHqax&R?Z z7nfRil0%V%(7?lDS4fxza76j|_)rrg=Etm>1qR6;LO}}Y27glKHv>{dPjK#KUP?Ah zSPSOga|aVLeD@8v3NodB<_4+;9PzFz-&|8_Y{1t`{6S}IH$ zo!mbN#dvm4`osF2Z0e3|G2V;{c8w#biT$0Sj>{7hpUgOreiVGblVj>>2gDD9Z!?(~ z?zu{RP-3B(f9m+u)VG{%Ak06;N+s@_pG8V9Rb*4n`?i{bLh`s!MaDAYueLQ`70ySD zF+X{1C3QA~F)4*8!)N~K(>^d?05?>fYqWTak~;@#5DRidJzp`y~pAP(&4kjs=En_D4kBZ9rh7KEn$t#riWef%~N%7HnLwv7DH z3_^qOXbk+ql75oIw&2iUg<{$};jO%!%XCf?N-C;%3rN{SqvbSt)*a(W0=)`MJFU!* zOp4F&cwu++P|8?D5+;lkOrm$6roO)q0G7`I`IE`1DMCLcOz1l~CY)1qD=WmW;#)Ke z>KVv6W5HhXYRbz}iQE|LZA13>e!lb?qr_PrD?|#d!*bMUbka9lw$U36ZEY-yGM1(@ z$n#8X;h}Jep#{I=|0C5($Ji?a`$Xt6 zNk|M14vN-+J_RBY_xJa4NvhSVa7^Bn>m>9O)6pff3iKdEfT?ggF6_7BE#afVMmSNZ zm5$#R)P&&q6ij16Hp_EiBFa@3wR2`>=IYg}OG`^#%0Sn>By_QAy12L~r(TTm=g*(N zdGki3oN!Vum{1NWH6jU7OD1o@r^@>-%Yq}XSAwsp8a8&R`p{hDSkd_;)HM*)Jhk#k z9#Y|iFhrCX<2ywFPE1TpPfsr_EXXpD>t6KvI6bS_~pp_NgBMt$%9ir$@DDH;f*gi$)a}FJ4?-T@@{KWMri1GtmTewdmjdxSC@8-qKSmX<^XE&3>%dd7Nk zJslhz{Q2jfPo6vx)l@hMgHV8x-GC+`w>1MvLh#e*qI%;C8`O6)f;wk@`g@e_Jz+@5 z(g!^t$wc9Gzj_G3$;n9(fWknLgh)^8CG;i<1wu2S;@aBU)2B~2H#e)*Duf|rpx{3- z&~=~U(b$KiC2d7Rzo4ro2aSBJoqWy@foA!rvPfvEAN-C?D1q07x%7YPVhG60p`oEl zr7|@&wYa#ru&{uUvb~3%B_VKpeEj_R^S}T8`^AeFZ{NNZbre(t4-vI5;@ihLck``h|pm$UtG^ zqeqXPJbALcy)E0qDld4QQL?zC(>Qa! zYqm|KLF=EvUi2ms2yK%!f%Jt68^_1TXJ= zQs>s7cLrZENj4_!#wl5+gB+S{K{K-H2istZnKAIVVd4#Z(a*is(vPtRoed)l2XLpTS&E*&ZDo75Y^wmHGMk%a<<;C;ORx|6GESgaF7UvhL~Ar;v)f zySo@yN|DG~VK9xHB=`8}tEQ&&VlWjhi-fgw|NpE*Xxs&Abj~aRsMD;YQ1f}+cTiK2 z5kI_$p1i}u!@|JX*;!$r&<_FnqAz<#pd_IMT%+qTL@J8iNF7xkkzP|vGHgImZ6fMN zM*NK=^j$cHgobiD;1P7L*7bAh+!vpC&TFznkBiGFdcenYtdGU-6o9OJ9~l`LA0MBd zo)-Fv3W|4&zRn$ll7wNP3<^5x#>R#SM$|*0b&c=}f>C==H4h;kU95KEb>Dg85|QXr zk)_6dP%#*I1PMtS^A(-Q0`QdqHS|h5U^EHII?Y26Nyj7*=wr%@yn-fZ6B84nd5YnO z_ynAyB{WJB<^ejYFzVT}XTrwq?QJm^4-O8Xl+t;l-KTWn=vbl-`UgpSZf;HlAR>W9U(C)$Ny3I8 z^blTwiaR?y`}_MsMKqYAhlXs_6~lh2O|XF2cvD?Ez-Uy^S}5*sNNB7+O+uf6QM!Aw zpZ2Lsjp%gIN!XZ%D^}mo(2zVr`4r8wKSK^DMM=VcAR1vQ>RXI1+i{vD3|%PTt|gvJlTfEt>$Fdj&?RZv zOgdwgtvm{^#Ogzr7q(-9KZb{|3no6tQ_%Hp%dd4;quy4ae@VEzyNia1APg3qc8&aV zf|AJ=rdz2~D@kOa(C_f@@Hhs_qbL}4F1QYFm^4%)2A!7?p2b6#cBmS+r%^jyBGaPK zz$1z~)BF>>LeI}LXU<^!7>LIngmk@fR*w#5a3?`{?)*SJ%uF`qQDkY+i=r6xDG5cB z6wWY3+j$iaWCcm6Get=ltSTL`&?(?-Mr^A1As)3D)S}X;qdDH1C_R9YM;A2tqKtI2 z0k7dFsPP8rS9mPkAM~+zF&jm?7QH~)ioPQux?NEJ)Fh0O zDNb7K=oo`>a=MKAdirUgml_7@gc~CX@z;3Ec>xTU&Z^;R*8A^5%BN0zaVqXNkTlrU zZcm*8%AQMl`U=R2zh?CyInQ}@x|lFZ7uxH~p><;hLPW;)l~nAnjG_1M-;3p@nMP@Y zb>Is_W3ZXXyGx5r`aeN1*gsg=##;j&*w}>Hm*YYtbQ#_nUj@}014(1gTDtALRz9^C z)X&KV>!=8jOE_=3pb2{(#dCPHbfw(ZHeuxDB;Y{`b!3j zJ3+TlP_%0AsG+Xl%$YNlN~P#E(r)w}2_eu(#nib_t`0-RV3i=?)5dQl%un)X^(%g) zUMrbK7sU$JkrEe5;PzR;PdZOh8SOxSjJet*p$l~rHK%dFtGb|+*1jCQ{}6y$N9`Z< z9SMQD_MR+-w{PEKkdvQf=Dane$!TO@F!)ZbOiNnIM&}u>TAUiLx_TsZ>HCY3U`ClV zI{847%NBX%g;`2(r2L$biWYwL)mIY}6Q47>d-!DTAM_~+8v)n7tRYlUxKK#IshVAv z7H&rFX*wCHtJ46m!4=HEfC8+4L@($_wHq{4^p#s*Sn1EZkl)qk9-Tc=+H=FV1_uYn z$H(>I_btuxQIfDB7~Ri#l91c7{!nS*Jh^8yOr!L25&M$RB_sr4g1I$R9l0m|9{M{I zd2xJ0R~{5TbN1}nv9Ynw^(KvH3FHz%lq75kPI^{lvF`5f3Uwnjq7-SF{)!h$?8VTW zi4qs?$jHd(=&1gdyk4zV<@fOcyePjczkiDTEc(>;AW9N;5X`m6U-4NNF6&z-`tcI& zL`lLDB??C0xr0lT=roihEK#Chlq4)sqF|IHEK#Ch{2yDykWdKcYZXhx;AWdO;ATlsAGaxZFIx;poF)$!2FflMNFqEPc00007bV*G` z2j>MA4IBuXb3l~<000SaNLh0L01FZT01FZU(%pXi00004XF*Lt006O%3;baP0007Q zP)t-s|NsB`)ynpnE`^rejLVKx8${{8aq z^qhd~UNHUd>HhZf{ORKS=HBpqS@^)M{qXDi-O=uHO7^dq|NQ#>?B)C3)a`9U_OqS- z`uOvcbnRj{`Np*T=;8LYp6+Ei^O14<Qf`{cvAGEi22jT`qIVmi)!|; zne(EE`N_HY%DVNclJusG?|@wMmU!}uYXAKF@{Miz(ZbwMBH&OWk24m%L>;t29NA4F zkTVv5; zulw1~{P_31hjxoXD~LZRw|#5#=Hlzv(zADDi9aZiNHM~Xe)Ob^@Z#O9ZCRmSN#@VT z-@>?W)H>g2_gg{x~<&!(FD-PFvXlfR35*0H7a zt(KHbHiSAOBqscBVrEEJ<*Ox3TW&ZU>1TSj~_ z8Ln_$&GqvN5B z%19sAbyn}gw7f$d-H~|iZ%62_oz;9`?O!s=WkkP8B<#AX`q#?ymwE4XP3>Sa>|HVK zYC-P5u=v{0^OAD;&A;7nOw~*v?XjKDa8m5Ntntaa)_-I4&%)S-YuAHm#8xuxy{`M? z+Vwy8m;e9(0d!JMQvg8b*k%9#17}G@K~#9!otEcclQ9s-!z%C;sMf+Sse+t1sbQT+6bg zn7QxTy7~rwDWdT$MU4^sYD0ZpEeDwSdr@UH8q?82bqxyi%UAdfc>M-_)>NbQbZA(` z-w~9jWkvxkuK?h|!@|l(kL}!(r-Gc^ZD^j07`I>CBb3=wcnaB_J|lu>&%t5g`3nFP;Gl*(4jahi0&pPj;Gz7(_NX2? zdhGZK_?^q(pq5JjJD6i}R39uk1Zd_`F2e>gOlH=8in22KRSwMHQjSaV)HECDTo6o4 z<&zwU<=4F=7mN+!MV5g(cJA7}XK!5czGNz>#9m_(+_H6BLgI#v+Y{ocpjvpWkU}M1 z#UpUjW^yu=x7FeAajXPfy=HCPy7iLDSo}rS3){!Ij!leUswYc8v@0%}ECKeS6B)Gv z(@JQ%3L8g7N{fzzGPewZ%V`T(+Y&q)kZ4*crF`)c2riu~npJ5ho0f|_+ zh**&{4`jPCYc>(gIoP&3d81?BbB6R7W`^7LlX3QzY}$0n2@0u3YstjePM$(N`>C>c z;zDibL/dev/null + +# Add Helm repos +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo add jetstack https://charts.jetstack.io + +# Update Helm repos +helm repo update + +if [[ $private == 'true' ]]; then + # Log whether the cluster is public or private + echo "$clusterName AKS cluster is public" + + # Install Prometheus + command="helm install prometheus prometheus-community/kube-prometheus-stack \ + --create-namespace \ + --namespace prometheus \ + --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ + --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false" + + az aks command invoke \ + --name $clusterName \ + --resource-group $resourceGroupName \ + --subscription $subscriptionId \ + --command "$command" + + # Install NGINX ingress controller using the internal load balancer + command="helm install nginx-ingress ingress-nginx/ingress-nginx \ + --create-namespace \ + --namespace ingress-basic \ + --set controller.replicaCount=3 \ + --set controller.nodeSelector.\"kubernetes\.io/os\"=linux \ + --set defaultBackend.nodeSelector.\"kubernetes\.io/os\"=linux \ + --set controller.metrics.enabled=true \ + --set controller.metrics.serviceMonitor.enabled=true \ + --set controller.metrics.serviceMonitor.additionalLabels.release=\"prometheus\" \ + --set controller.service.annotations.\"service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path\"=/healthz" + + az aks command invoke \ + --name $clusterName \ + --resource-group $resourceGroupName \ + --subscription $subscriptionId \ + --command "$command" + + # Install certificate manager + command="helm install cert-manager jetstack/cert-manager \ + --create-namespace \ + --namespace cert-manager \ + --set installCRDs=true \ + --set nodeSelector.\"kubernetes\.io/os\"=linux" + + az aks command invoke \ + --name $clusterName \ + --resource-group $resourceGroupName \ + --subscription $subscriptionId \ + --command "$command" + + # Create cluster issuer + command="cat <$AZ_SCRIPTS_OUTPUT_PATH \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf new file mode 100644 index 000000000..e8ed5536b --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -0,0 +1,454 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "3.58" + } + } +} + +provider "azurerm" { + features {} +} + +locals { + storage_account_prefix = "boot" +} + +data "azurerm_client_config" "current" { +} + +resource "random_string" "prefix" { + length = 6 + special = false + upper = false + numeric = false +} + +resource "random_string" "storage_account_suffix" { + length = 8 + special = false + lower = true + upper = false + numeric = false +} + +resource "azurerm_resource_group" "rg" { + name = var.name_prefix == null ? "${random_string.prefix.result}${var.resource_group_name}" : "${var.name_prefix}${var.resource_group_name}" + location = var.location + tags = var.tags +} + +module "log_analytics_workspace" { + source = "./modules/log_analytics" + name = var.name_prefix == null ? "${random_string.prefix.result}${var.log_analytics_workspace_name}" : "${var.name_prefix}${var.log_analytics_workspace_name}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + solution_plan_map = var.solution_plan_map + tags = var.tags +} + +module "virtual_network" { + source = "./modules/virtual_network" + resource_group_name = azurerm_resource_group.rg.name + location = var.location + vnet_name = var.name_prefix == null ? "${random_string.prefix.result}${var.vnet_name}" : "${var.name_prefix}${var.vnet_name}" + address_space = var.vnet_address_space + log_analytics_workspace_id = module.log_analytics_workspace.id + log_analytics_retention_days = var.log_analytics_retention_days + tags = var.tags + + subnets = [ + { + name : var.system_node_pool_subnet_name + address_prefixes : var.system_node_pool_subnet_address_prefix + private_endpoint_network_policies_enabled : true + private_link_service_network_policies_enabled : false + delegation: null + }, + { + name : var.user_node_pool_subnet_name + address_prefixes : var.user_node_pool_subnet_address_prefix + private_endpoint_network_policies_enabled : true + private_link_service_network_policies_enabled : false + delegation: null + }, + { + name : var.pod_subnet_name + address_prefixes : var.pod_subnet_address_prefix + private_endpoint_network_policies_enabled : true + private_link_service_network_policies_enabled : false + delegation: "Microsoft.ContainerService/managedClusters" + }, + { + name : var.vm_subnet_name + address_prefixes : var.vm_subnet_address_prefix + private_endpoint_network_policies_enabled : true + private_link_service_network_policies_enabled : false + delegation: null + }, + { + name : "AzureBastionSubnet" + address_prefixes : var.bastion_subnet_address_prefix + private_endpoint_network_policies_enabled : true + private_link_service_network_policies_enabled : false + delegation: null + } + ] +} + +module "nat_gateway" { + source = "./modules/nat_gateway" + name = var.name_prefix == null ? "${random_string.prefix.result}${var.nat_gateway_name}" : "${var.name_prefix}${var.nat_gateway_name}" + resource_group_name = azurerm_resource_group.rg.name + location = var.location + sku_name = var.nat_gateway_sku_name + idle_timeout_in_minutes = var.nat_gateway_idle_timeout_in_minutes + zones = var.nat_gateway_zones + tags = var.tags + subnet_ids = module.virtual_network.subnet_ids +} + +module "container_registry" { + source = "./modules/container_registry" + name = var.name_prefix == null ? "${random_string.prefix.result}${var.acr_name}" : "${var.name_prefix}${var.acr_name}" + resource_group_name = azurerm_resource_group.rg.name + location = var.location + sku = var.acr_sku + admin_enabled = var.acr_admin_enabled + georeplication_locations = var.acr_georeplication_locations + log_analytics_workspace_id = module.log_analytics_workspace.id + log_analytics_retention_days = var.log_analytics_retention_days + tags = var.tags + +} + +module "aks_cluster" { + source = "./modules/aks" + name = var.name_prefix == null ? "${random_string.prefix.result}${var.aks_cluster_name}" : "${var.name_prefix}${var.aks_cluster_name}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + resource_group_id = azurerm_resource_group.rg.id + kubernetes_version = var.kubernetes_version + dns_prefix = lower(var.aks_cluster_name) + private_cluster_enabled = var.private_cluster_enabled + automatic_channel_upgrade = var.automatic_channel_upgrade + sku_tier = var.sku_tier + system_node_pool_name = var.system_node_pool_name + system_node_pool_vm_size = var.system_node_pool_vm_size + vnet_subnet_id = module.virtual_network.subnet_ids[var.system_node_pool_subnet_name] + pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] + system_node_pool_availability_zones = var.system_node_pool_availability_zones + system_node_pool_node_labels = var.system_node_pool_node_labels + system_node_pool_node_taints = var.system_node_pool_node_taints + system_node_pool_enable_auto_scaling = var.system_node_pool_enable_auto_scaling + system_node_pool_enable_host_encryption = var.system_node_pool_enable_host_encryption + system_node_pool_enable_node_public_ip = var.system_node_pool_enable_node_public_ip + system_node_pool_max_pods = var.system_node_pool_max_pods + system_node_pool_max_count = var.system_node_pool_max_count + system_node_pool_min_count = var.system_node_pool_min_count + system_node_pool_node_count = var.system_node_pool_node_count + system_node_pool_os_disk_type = var.system_node_pool_os_disk_type + tags = var.tags + network_dns_service_ip = var.network_dns_service_ip + network_plugin = var.network_plugin + outbound_type = "userAssignedNATGateway" + network_service_cidr = var.network_service_cidr + log_analytics_workspace_id = module.log_analytics_workspace.id + role_based_access_control_enabled = var.role_based_access_control_enabled + tenant_id = data.azurerm_client_config.current.tenant_id + admin_group_object_ids = var.admin_group_object_ids + azure_rbac_enabled = var.azure_rbac_enabled + admin_username = var.admin_username + ssh_public_key = var.ssh_public_key + keda_enabled = var.keda_enabled + vertical_pod_autoscaler_enabled = var.vertical_pod_autoscaler_enabled + workload_identity_enabled = var.workload_identity_enabled + oidc_issuer_enabled = var.oidc_issuer_enabled + open_service_mesh_enabled = var.open_service_mesh_enabled + image_cleaner_enabled = var.image_cleaner_enabled + azure_policy_enabled = var.azure_policy_enabled + http_application_routing_enabled = var.http_application_routing_enabled + + depends_on = [ + module.nat_gateway, + module.container_registry + ] +} + +module "node_pool" { + source = "./modules/node_pool" + resource_group_name = azurerm_resource_group.rg.name + kubernetes_cluster_id = module.aks_cluster.id + name = var.user_node_pool_name + vm_size = var.user_node_pool_vm_size + mode = var.user_node_pool_mode + node_labels = var.user_node_pool_node_labels + node_taints = var.user_node_pool_node_taints + availability_zones = var.user_node_pool_availability_zones + vnet_subnet_id = module.virtual_network.subnet_ids[var.user_node_pool_subnet_name] + pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] + enable_auto_scaling = var.user_node_pool_enable_auto_scaling + enable_host_encryption = var.user_node_pool_enable_host_encryption + enable_node_public_ip = var.user_node_pool_enable_node_public_ip + orchestrator_version = var.kubernetes_version + max_pods = var.user_node_pool_max_pods + max_count = var.user_node_pool_max_count + min_count = var.user_node_pool_min_count + node_count = var.user_node_pool_node_count + os_type = var.user_node_pool_os_type + priority = var.user_node_pool_priority + tags = var.tags +} + +module "openai" { + source = "./modules/openai" + name = var.name_prefix == null ? "${random_string.prefix.result}${var.openai_name}" : "${var.name_prefix}${var.openai_name}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + sku_name = var.openai_sku_name + tags = var.tags + deployments = var.openai_deployments + custom_subdomain_name = var.openai_custom_subdomain_name == "" || var.openai_custom_subdomain_name == null ? var.name_prefix == null ? lower("${random_string.prefix.result}${var.openai_name}") : lower("${var.name_prefix}${var.openai_name}") : lower(var.openai_custom_subdomain_name) + public_network_access_enabled = var.openai_public_network_access_enabled + log_analytics_workspace_id = module.log_analytics_workspace.id + log_analytics_retention_days = var.log_analytics_retention_days +} + +resource "azurerm_user_assigned_identity" "aks_workload_identity" { + name = var.name_prefix == null ? "${random_string.prefix.result}${var.workload_managed_identity_name}" : "${var.name_prefix}${var.workload_managed_identity_name}" + resource_group_name = azurerm_resource_group.rg.name + location = var.location + tags = var.tags + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_role_assignment" "cognitive_services_user_assignment" { + scope = module.openai.id + role_definition_name = "Cognitive Services User" + principal_id = azurerm_user_assigned_identity.aks_workload_identity.principal_id + skip_service_principal_aad_check = true +} + +resource "azurerm_federated_identity_credential" "federated_identity_credential" { + name = "${title(var.namespace)}FederatedIdentity" + resource_group_name = azurerm_resource_group.rg.name + audience = ["api://AzureADTokenExchange"] + issuer = module.aks_cluster.oidc_issuer_url + parent_id = azurerm_user_assigned_identity.aks_workload_identity.id + subject = "system:serviceaccount:${var.namespace}:${var.service_account_name}" +} + +resource "azurerm_role_assignment" "network_contributor_assignment" { + scope = azurerm_resource_group.rg.id + role_definition_name = "Network Contributor" + principal_id = module.aks_cluster.aks_identity_principal_id + skip_service_principal_aad_check = true +} + +resource "azurerm_role_assignment" "acr_pull_assignment" { + role_definition_name = "AcrPull" + scope = module.container_registry.id + principal_id = module.aks_cluster.kubelet_identity_object_id + skip_service_principal_aad_check = true +} + +module "storage_account" { + source = "./modules/storage_account" + name = "${local.storage_account_prefix}${random_string.storage_account_suffix.result}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + account_kind = var.storage_account_kind + account_tier = var.storage_account_tier + replication_type = var.storage_account_replication_type + tags = var.tags + +} + +module "bastion_host" { + source = "./modules/bastion_host" + name = var.name_prefix == null ? "${random_string.prefix.result}${var.bastion_host_name}" : "${var.name_prefix}${var.bastion_host_name}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + subnet_id = module.virtual_network.subnet_ids["AzureBastionSubnet"] + log_analytics_workspace_id = module.log_analytics_workspace.id + log_analytics_retention_days = var.log_analytics_retention_days + tags = var.tags +} + +module "virtual_machine" { + count = var.vm_enabled ? 1 : 0 + source = "./modules/virtual_machine" + name = var.name_prefix == null ? "${random_string.prefix.result}${var.vm_name}" : "${var.name_prefix}${var.vm_name}" + size = var.vm_size + location = var.location + public_ip = var.vm_public_ip + vm_user = var.admin_username + admin_ssh_public_key = var.ssh_public_key + os_disk_image = var.vm_os_disk_image + resource_group_name = azurerm_resource_group.rg.name + subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] + os_disk_storage_account_type = var.vm_os_disk_storage_account_type + boot_diagnostics_storage_account = module.storage_account.primary_blob_endpoint + log_analytics_workspace_id = module.log_analytics_workspace.workspace_id + log_analytics_workspace_key = module.log_analytics_workspace.primary_shared_key + log_analytics_workspace_resource_id = module.log_analytics_workspace.id + log_analytics_retention_days = var.log_analytics_retention_days + tags = var.tags +} + +module "key_vault" { + source = "./modules/key_vault" + name = var.name_prefix == null ? "${random_string.prefix.result}${var.key_vault_name}" : "${var.name_prefix}${var.key_vault_name}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + tenant_id = data.azurerm_client_config.current.tenant_id + sku_name = var.key_vault_sku_name + enabled_for_deployment = var.key_vault_enabled_for_deployment + enabled_for_disk_encryption = var.key_vault_enabled_for_disk_encryption + enabled_for_template_deployment = var.key_vault_enabled_for_template_deployment + enable_rbac_authorization = var.key_vault_enable_rbac_authorization + purge_protection_enabled = var.key_vault_purge_protection_enabled + soft_delete_retention_days = var.key_vault_soft_delete_retention_days + bypass = var.key_vault_bypass + default_action = var.key_vault_default_action + log_analytics_workspace_id = module.log_analytics_workspace.id + log_analytics_retention_days = var.log_analytics_retention_days + tags = var.tags +} + +module "acr_private_dns_zone" { + source = "./modules/private_dns_zone" + name = "privatelink.azurecr.io" + resource_group_name = azurerm_resource_group.rg.name + tags = var.tags + virtual_networks_to_link = { + (module.virtual_network.name) = { + subscription_id = data.azurerm_client_config.current.subscription_id + resource_group_name = azurerm_resource_group.rg.name + } + } +} + +module "openai_private_dns_zone" { + source = "./modules/private_dns_zone" + name = "privatelink.openai.azure.com" + resource_group_name = azurerm_resource_group.rg.name + tags = var.tags + virtual_networks_to_link = { + (module.virtual_network.name) = { + subscription_id = data.azurerm_client_config.current.subscription_id + resource_group_name = azurerm_resource_group.rg.name + } + } +} + +module "key_vault_private_dns_zone" { + source = "./modules/private_dns_zone" + name = "privatelink.vaultcore.azure.net" + resource_group_name = azurerm_resource_group.rg.name + tags = var.tags + virtual_networks_to_link = { + (module.virtual_network.name) = { + subscription_id = data.azurerm_client_config.current.subscription_id + resource_group_name = azurerm_resource_group.rg.name + } + } +} + +module "blob_private_dns_zone" { + source = "./modules/private_dns_zone" + name = "privatelink.blob.core.windows.net" + resource_group_name = azurerm_resource_group.rg.name + tags = var.tags + virtual_networks_to_link = { + (module.virtual_network.name) = { + subscription_id = data.azurerm_client_config.current.subscription_id + resource_group_name = azurerm_resource_group.rg.name + } + } +} + +module "openai_private_endpoint" { + source = "./modules/private_endpoint" + name = "${module.openai.name}PrivateEndpoint" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] + tags = var.tags + private_connection_resource_id = module.openai.id + is_manual_connection = false + subresource_name = "account" + private_dns_zone_group_name = "AcrPrivateDnsZoneGroup" + private_dns_zone_group_ids = [module.openai_private_dns_zone.id] +} + +module "acr_private_endpoint" { + source = "./modules/private_endpoint" + name = "${module.container_registry.name}PrivateEndpoint" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] + tags = var.tags + private_connection_resource_id = module.container_registry.id + is_manual_connection = false + subresource_name = "registry" + private_dns_zone_group_name = "AcrPrivateDnsZoneGroup" + private_dns_zone_group_ids = [module.acr_private_dns_zone.id] +} + +module "key_vault_private_endpoint" { + source = "./modules/private_endpoint" + name = "${module.key_vault.name}PrivateEndpoint" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] + tags = var.tags + private_connection_resource_id = module.key_vault.id + is_manual_connection = false + subresource_name = "vault" + private_dns_zone_group_name = "KeyVaultPrivateDnsZoneGroup" + private_dns_zone_group_ids = [module.key_vault_private_dns_zone.id] +} + +module "blob_private_endpoint" { + source = "./modules/private_endpoint" + name = var.name_prefix == null ? "${random_string.prefix.result}BlocStoragePrivateEndpoint" : "${var.name_prefix}BlobStoragePrivateEndpoint" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] + tags = var.tags + private_connection_resource_id = module.storage_account.id + is_manual_connection = false + subresource_name = "blob" + private_dns_zone_group_name = "BlobPrivateDnsZoneGroup" + private_dns_zone_group_ids = [module.blob_private_dns_zone.id] +} + +module "deployment_script" { + source = "./modules/deployment_script" + name = var.name_prefix == null ? "${random_string.prefix.result}${var.deployment_script_name}" : "${var.name_prefix}${var.deployment_script_name}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + azure_cli_version = var.deployment_script_azure_cli_version + managed_identity_name = var.name_prefix == null ? "${random_string.prefix.result}${var.deployment_script_managed_identity_name}" : "${var.name_prefix}${var.deployment_script_managed_identity_name}" + aks_cluster_name = module.aks_cluster.name + hostname = "${var.subdomain}.${var.domain}" + namespace = var.namespace + service_account_name = var.service_account_name + email = var.email + primary_script_uri = var.deployment_script_primary_script_uri + tenant_id = data.azurerm_client_config.current.tenant_id + subscription_id = data.azurerm_client_config.current.subscription_id + workload_managed_identity_client_id = azurerm_user_assigned_identity.aks_workload_identity.client_id + tags = var.tags + + depends_on = [ + module.aks_cluster + ] +} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf new file mode 100644 index 000000000..49a6622a6 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -0,0 +1,180 @@ +resource "azurerm_user_assigned_identity" "aks_identity" { + resource_group_name = var.resource_group_name + location = var.location + tags = var.tags + + name = "${var.name}Identity" + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_kubernetes_cluster" "aks_cluster" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + kubernetes_version = var.kubernetes_version + dns_prefix = var.dns_prefix + private_cluster_enabled = var.private_cluster_enabled + automatic_channel_upgrade = var.automatic_channel_upgrade + sku_tier = var.sku_tier + workload_identity_enabled = var.workload_identity_enabled + oidc_issuer_enabled = var.oidc_issuer_enabled + open_service_mesh_enabled = var.open_service_mesh_enabled + image_cleaner_enabled = var.image_cleaner_enabled + azure_policy_enabled = var.azure_policy_enabled + http_application_routing_enabled = var.http_application_routing_enabled + + default_node_pool { + name = var.system_node_pool_name + vm_size = var.system_node_pool_vm_size + vnet_subnet_id = var.vnet_subnet_id + pod_subnet_id = var.pod_subnet_id + zones = var.system_node_pool_availability_zones + node_labels = var.system_node_pool_node_labels + node_taints = var.system_node_pool_node_taints + enable_auto_scaling = var.system_node_pool_enable_auto_scaling + enable_host_encryption = var.system_node_pool_enable_host_encryption + enable_node_public_ip = var.system_node_pool_enable_node_public_ip + max_pods = var.system_node_pool_max_pods + max_count = var.system_node_pool_max_count + min_count = var.system_node_pool_min_count + node_count = var.system_node_pool_node_count + os_disk_type = var.system_node_pool_os_disk_type + tags = var.tags + } + + linux_profile { + admin_username = var.admin_username + ssh_key { + key_data = var.ssh_public_key + } + } + + identity { + type = "UserAssigned" + identity_ids = tolist([azurerm_user_assigned_identity.aks_identity.id]) + } + + network_profile { + dns_service_ip = var.network_dns_service_ip + network_plugin = var.network_plugin + outbound_type = var.outbound_type + service_cidr = var.network_service_cidr + } + + oms_agent { + msi_auth_for_monitoring_enabled = true + log_analytics_workspace_id = coalesce(var.oms_agent.log_analytics_workspace_id, var.log_analytics_workspace_id) + } + + dynamic "ingress_application_gateway" { + for_each = try(var.ingress_application_gateway.gateway_id, null) == null ? [] : [1] + + content { + gateway_id = var.ingress_application_gateway.gateway_id + subnet_cidr = var.ingress_application_gateway.subnet_cidr + subnet_id = var.ingress_application_gateway.subnet_id + } + } + + azure_active_directory_role_based_access_control { + managed = true + tenant_id = var.tenant_id + admin_group_object_ids = var.admin_group_object_ids + azure_rbac_enabled = var.azure_rbac_enabled + } + + workload_autoscaler_profile { + keda_enabled = var.keda_enabled + vertical_pod_autoscaler_enabled = var.vertical_pod_autoscaler_enabled + } + + lifecycle { + ignore_changes = [ + kubernetes_version, + tags + ] + } +} + +resource "azurerm_monitor_diagnostic_setting" "settings" { + name = "DiagnosticsSettings" + target_resource_id = azurerm_kubernetes_cluster.aks_cluster.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "kube-apiserver" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "kube-audit" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "kube-audit-admin" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "kube-controller-manager" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "kube-scheduler" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "cluster-autoscaler" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "guard" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + metric { + category = "AllMetrics" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf new file mode 100644 index 000000000..576a7399d --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf @@ -0,0 +1,40 @@ +output "name" { + value = azurerm_kubernetes_cluster.aks_cluster.name + description = "Specifies the name of the AKS cluster." +} + +output "id" { + value = azurerm_kubernetes_cluster.aks_cluster.id + description = "Specifies the resource id of the AKS cluster." +} + + +output "aks_identity_principal_id" { + value = azurerm_user_assigned_identity.aks_identity.principal_id + description = "Specifies the principal id of the managed identity of the AKS cluster." +} + +output "kubelet_identity_object_id" { + value = azurerm_kubernetes_cluster.aks_cluster.kubelet_identity.0.object_id + description = "Specifies the object id of the kubelet identity of the AKS cluster." +} + +output "kube_config_raw" { + value = azurerm_kubernetes_cluster.aks_cluster.kube_config_raw + description = "Contains the Kubernetes config to be used by kubectl and other compatible tools." +} + +output "private_fqdn" { + value = azurerm_kubernetes_cluster.aks_cluster.private_fqdn + description = "The FQDN for the Kubernetes Cluster when private link has been enabled, which is only resolvable inside the Virtual Network used by the Kubernetes Cluster." +} + +output "node_resource_group" { + value = azurerm_kubernetes_cluster.aks_cluster.node_resource_group + description = "Specifies the resource id of the auto-generated Resource Group which contains the resources for this Managed Kubernetes Cluster." +} + +output "oidc_issuer_url" { + value = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url + description = "Specifies the URL of the OpenID Connect issuer used by this Kubernetes Cluster." +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf new file mode 100644 index 000000000..33c66482b --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf @@ -0,0 +1,316 @@ +variable "name" { + description = "(Required) Specifies the name of the AKS cluster." + type = string +} + +variable "resource_group_name" { + description = "(Required) Specifies the name of the resource group." + type = string +} + +variable "resource_group_id" { + description = "(Required) Specifies the resource id of the resource group." + type = string +} + +variable "location" { + description = "(Required) Specifies the location where the AKS cluster will be deployed." + type = string +} + +variable "dns_prefix" { + description = "(Optional) DNS prefix specified when creating the managed cluster. Changing this forces a new resource to be created." + type = string +} + +variable "private_cluster_enabled" { + description = "Should this Kubernetes Cluster have its API server only exposed on internal IP addresses? This provides a Private IP Address for the Kubernetes API on the Virtual Network where the Kubernetes Cluster is located. Defaults to false. Changing this forces a new resource to be created." + type = bool + default = false +} + +variable "azure_rbac_enabled" { + description = "(Optional) Is Role Based Access Control based on Azure AD enabled?" + default = true + type = bool +} + +variable "admin_group_object_ids" { + description = "(Optional) A list of Object IDs of Azure Active Directory Groups which should have Admin Role on the Cluster." + default = [] + type = list(string) +} + +variable "role_based_access_control_enabled" { + description = "(Required) Is Role Based Access Control Enabled? Changing this forces a new resource to be created." + default = true + type = bool +} + +variable "automatic_channel_upgrade" { + description = "(Optional) The upgrade channel for this Kubernetes Cluster. Possible values are patch, rapid, and stable." + default = "stable" + type = string + + validation { + condition = contains( ["patch", "rapid", "stable"], var.automatic_channel_upgrade) + error_message = "The upgrade mode is invalid." + } +} + +variable "sku_tier" { + description = "(Optional) The SKU Tier that should be used for this Kubernetes Cluster. Possible values are Free and Paid (which includes the Uptime SLA). Defaults to Free." + default = "Free" + type = string + + validation { + condition = contains( ["Free", "Paid"], var.sku_tier) + error_message = "The sku tier is invalid." + } +} + +variable "kubernetes_version" { + description = "Specifies the AKS Kubernetes version" + default = "1.21.1" + type = string +} + +variable "system_node_pool_vm_size" { + description = "Specifies the vm size of the system node pool" + default = "Standard_F8s_v2" + type = string +} + +variable "system_node_pool_availability_zones" { + description = "Specifies the availability zones of the system node pool" + default = ["1", "2", "3"] + type = list(string) +} + +variable "network_dns_service_ip" { + description = "Specifies the DNS service IP" + default = "10.2.0.10" + type = string +} + +variable "network_service_cidr" { + description = "Specifies the service CIDR" + default = "10.2.0.0/24" + type = string +} + +variable "network_plugin" { + description = "Specifies the network plugin of the AKS cluster" + default = "azure" + type = string +} + +variable "outbound_type" { + description = "(Optional) The outbound (egress) routing method which should be used for this Kubernetes Cluster. Possible values are loadBalancer and userDefinedRouting. Defaults to loadBalancer." + type = string + default = "userDefinedRouting" + + validation { + condition = contains(["loadBalancer", "userDefinedRouting", "userAssignedNATGateway", "managedNATGateway"], var.outbound_type) + error_message = "The outbound type is invalid." + } +} + +variable "system_node_pool_name" { + description = "Specifies the name of the system node pool" + default = "system" + type = string +} + +variable "system_node_pool_subnet_name" { + description = "Specifies the name of the subnet that hosts the system node pool" + default = "SystemSubnet" + type = string +} + +variable "system_node_pool_subnet_address_prefix" { + description = "Specifies the address prefix of the subnet that hosts the system node pool" + default = ["10.0.0.0/20"] + type = list(string) +} + +variable "system_node_pool_enable_auto_scaling" { + description = "(Optional) Whether to enable auto-scaler. Defaults to false." + type = bool + default = true +} + +variable "system_node_pool_enable_host_encryption" { + description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." + type = bool + default = false +} + +variable "system_node_pool_enable_node_public_ip" { + description = "(Optional) Should each node have a Public IP Address? Defaults to false. Changing this forces a new resource to be created." + type = bool + default = false +} + +variable "system_node_pool_max_pods" { + description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." + type = number + default = 50 +} + +variable "system_node_pool_node_labels" { + description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." + type = map(any) + default = {} +} + +variable "system_node_pool_node_taints" { + description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." + type = list(string) + default = [] +} + +variable "system_node_pool_os_disk_type" { + description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." + type = string + default = "Ephemeral" +} + +variable "system_node_pool_max_count" { + description = "(Required) The maximum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be greater than or equal to min_count." + type = number + default = 10 +} + +variable "system_node_pool_min_count" { + description = "(Required) The minimum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be less than or equal to max_count." + type = number + default = 3 +} + +variable "system_node_pool_node_count" { + description = "(Optional) The initial number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be a value in the range min_count - max_count." + type = number + default = 3 +} + +variable "log_analytics_workspace_id" { + description = "(Optional) The ID of the Log Analytics Workspace which the OMS Agent should send data to. Must be present if enabled is true." + type = string +} + +variable "tenant_id" { + description = "(Required) The tenant id of the system assigned identity which is used by master components." + type = string +} + +variable "log_analytics_retention_days" { + description = "Specifies the number of days of the retention policy" + type = number + default = 30 +} + +variable "vnet_subnet_id" { + description = "(Optional) The ID of a Subnet where the Kubernetes Node Pool should exist. Changing this forces a new resource to be created." + type = string +} + +variable "pod_subnet_id" { + description = "(Optional) The ID of the Subnet where the pods in the system node pool should exist. Changing this forces a new resource to be created." + type = string + default = null +} + +variable "tags" { + description = "(Optional) Specifies the tags of the bastion host" + default = {} +} + +variable "oms_agent" { + description = "Specifies the OMS agent addon configuration." + type = object({ + enabled = bool + log_analytics_workspace_id = string + }) + default = { + enabled = true + log_analytics_workspace_id = null + } +} + +variable "ingress_application_gateway" { + description = "Specifies the Application Gateway Ingress Controller addon configuration." + type = object({ + enabled = bool + gateway_id = string + gateway_name = string + subnet_cidr = string + subnet_id = string + }) + default = { + enabled = false + gateway_id = null + gateway_name = null + subnet_cidr = null + subnet_id = null + } +} + +variable "admin_username" { + description = "(Required) Specifies the Admin Username for the AKS cluster worker nodes. Changing this forces a new resource to be created." + type = string + default = "azadmin" +} + +variable "ssh_public_key" { + description = "(Required) Specifies the SSH public key used to access the cluster. Changing this forces a new resource to be created." + type = string +} + +variable "keda_enabled" { + description = "(Optional) Specifies whether KEDA Autoscaler can be used for workloads." + type = bool + default = true +} + +variable "vertical_pod_autoscaler_enabled" { + description = "(Optional) Specifies whether Vertical Pod Autoscaler should be enabled." + type = bool + default = true +} + +variable "workload_identity_enabled" { + description = "(Optional) Specifies whether Azure AD Workload Identity should be enabled for the Cluster. Defaults to false." + type = bool + default = true +} + +variable "oidc_issuer_enabled" { + description = "(Optional) Enable or Disable the OIDC issuer URL." + type = bool + default = true +} + +variable "open_service_mesh_enabled" { + description = "(Optional) Is Open Service Mesh enabled? For more details, please visit Open Service Mesh for AKS." + type = bool + default = true +} + +variable "image_cleaner_enabled" { + description = "(Optional) Specifies whether Image Cleaner is enabled." + type = bool + default = true +} + +variable "azure_policy_enabled" { + description = "(Optional) Should the Azure Policy Add-On be enabled? For more details please visit Understand Azure Policy for Azure Kubernetes Service" + type = bool + default = true +} + +variable "http_application_routing_enabled" { + description = "(Optional) Should HTTP Application Routing be enabled?" + type = bool + default = false +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf new file mode 100644 index 000000000..34a01d40b --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf @@ -0,0 +1,99 @@ +resource "azurerm_public_ip" "public_ip" { + name = "${var.name}PublicIp" + location = var.location + resource_group_name = var.resource_group_name + allocation_method = "Static" + sku = "Standard" + tags = var.tags + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_bastion_host" "bastion_host" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + tags = var.tags + + ip_configuration { + name = "configuration" + subnet_id = var.subnet_id + public_ip_address_id = azurerm_public_ip.public_ip.id + } + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_monitor_diagnostic_setting" "settings" { + name = "DiagnosticsSettings" + target_resource_id = azurerm_bastion_host.bastion_host.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "BastionAuditLogs" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + metric { + category = "AllMetrics" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } +} + +resource "azurerm_monitor_diagnostic_setting" "pip_settings" { + name = "DiagnosticsSettings" + target_resource_id = azurerm_public_ip.public_ip.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "DDoSProtectionNotifications" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "DDoSMitigationFlowLogs" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "DDoSMitigationReports" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + metric { + category = "AllMetrics" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/output.tf new file mode 100644 index 000000000..91b9f9386 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/output.tf @@ -0,0 +1,23 @@ +output "name" { + depends_on = [azurerm_bastion_host.bastion_host] + value = azurerm_bastion_host.bastion_host.*.name + description = "Specifies the name of the bastion host" +} + +output "id" { + depends_on = [azurerm_bastion_host.bastion_host] + value = azurerm_bastion_host.bastion_host.*.id + description = "Specifies the resource id of the bastion host" +} + +output "bastion_host" { + depends_on = [azurerm_bastion_host.bastion_host] + value = azurerm_bastion_host.bastion_host + description = "Contains the bastion host resource" +} + +output "public_ip_address" { + depends_on = [azurerm_bastion_host.bastion_host] + value = azurerm_public_ip.public_ip.ip_address + description = "Contains the public IP address of the bastion host." +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf new file mode 100644 index 000000000..77f686eed --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf @@ -0,0 +1,35 @@ +variable "resource_group_name" { + description = "(Required) Specifies the resource group name of the bastion host" + type = string +} + +variable "name" { + description = "(Required) Specifies the name of the bastion host" + type = string +} + +variable "location" { + description = "(Required) Specifies the location of the bastion host" + type = string +} + +variable "tags" { + description = "(Optional) Specifies the tags of the bastion host" + default = {} +} + +variable "subnet_id" { + description = "(Required) Specifies subnet id of the bastion host" + type = string +} + +variable "log_analytics_workspace_id" { + description = "Specifies the log analytics workspace id" + type = string +} + +variable "log_analytics_retention_days" { + description = "Specifies the number of days of the retention policy" + type = number + default = 7 +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf new file mode 100644 index 000000000..38e3b49f3 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf @@ -0,0 +1,77 @@ +resource "azurerm_container_registry" "acr" { + name = var.name + resource_group_name = var.resource_group_name + location = var.location + sku = var.sku + admin_enabled = var.admin_enabled + tags = var.tags + + identity { + type = "UserAssigned" + identity_ids = [ + azurerm_user_assigned_identity.acr_identity.id + ] + } + + dynamic "georeplications" { + for_each = var.georeplication_locations + + content { + location = georeplications.value + tags = var.tags + } + } + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_user_assigned_identity" "acr_identity" { + resource_group_name = var.resource_group_name + location = var.location + tags = var.tags + + name = "${var.name}Identity" + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_monitor_diagnostic_setting" "settings" { + name = "DiagnosticsSettings" + target_resource_id = azurerm_container_registry.acr.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "ContainerRegistryRepositoryEvents" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "ContainerRegistryLoginEvents" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + metric { + category = "AllMetrics" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf new file mode 100644 index 000000000..1834bc59c --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf @@ -0,0 +1,29 @@ +output "name" { + description = "Specifies the name of the container registry." + value = azurerm_container_registry.acr.name +} + +output "id" { + description = "Specifies the resource id of the container registry." + value = azurerm_container_registry.acr.id +} + +output "resource_group_name" { + description = "Specifies the name of the resource group." + value = var.resource_group_name +} + +output "login_server" { + description = "Specifies the login server of the container registry." + value = azurerm_container_registry.acr.login_server +} + +output "login_server_url" { + description = "Specifies the login server url of the container registry." + value = "https://${azurerm_container_registry.acr.login_server}" +} + +output "admin_username" { + description = "Specifies the admin username of the container registry." + value = azurerm_container_registry.acr.admin_username +} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf new file mode 100644 index 000000000..3bf6ae317 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf @@ -0,0 +1,54 @@ +variable "name" { + description = "(Required) Specifies the name of the Container Registry. Changing this forces a new resource to be created." + type = string +} + +variable "resource_group_name" { + description = "(Required) The name of the resource group in which to create the Container Registry. Changing this forces a new resource to be created." + type = string +} + +variable "location" { + description = "(Required) Specifies the supported Azure location where the resource exists. Changing this forces a new resource to be created." + type = string +} + +variable "admin_enabled" { + description = "(Optional) Specifies whether the admin user is enabled. Defaults to false." + type = string + default = false +} + +variable "sku" { + description = "(Optional) The SKU name of the container registry. Possible values are Basic, Standard and Premium. Defaults to Basic" + type = string + default = "Basic" + + validation { + condition = contains(["Basic", "Standard", "Premium"], var.sku) + error_message = "The container registry sku is invalid." + } +} + +variable "tags" { + description = "(Optional) A mapping of tags to assign to the resource." + type = map(any) + default = {} +} + +variable "georeplication_locations" { + description = "(Optional) A list of Azure locations where the container registry should be geo-replicated." + type = list(string) + default = [] +} + +variable "log_analytics_workspace_id" { + description = "Specifies the log analytics workspace id" + type = string +} + +variable "log_analytics_retention_days" { + description = "Specifies the number of days of the retention policy" + type = number + default = 7 +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf new file mode 100644 index 000000000..e5f05b5f8 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf @@ -0,0 +1,95 @@ +resource "azurerm_user_assigned_identity" "script_identity" { + name = var.managed_identity_name + location = var.location + resource_group_name = var.resource_group_name + tags = var.tags + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +data "azurerm_kubernetes_cluster" "aks_cluster" { + name = var.aks_cluster_name + resource_group_name = var.resource_group_name +} + +resource "azurerm_role_assignment" "network_contributor_assignment" { + scope = data.azurerm_kubernetes_cluster.aks_cluster.id + role_definition_name = "Azure Kubernetes Service Cluster Admin Role" + principal_id = azurerm_user_assigned_identity.script_identity.principal_id + skip_service_principal_aad_check = true +} + +resource "azurerm_resource_deployment_script_azure_cli" "script" { + name = var.name + resource_group_name = var.resource_group_name + location = var.location + version = var.azure_cli_version + retention_interval = "P1D" + command_line = "'foo' 'bar'" + cleanup_preference = "OnSuccess" + force_update_tag = "1" + timeout = "PT30M" + primary_script_uri = var.primary_script_uri + tags = var.tags + + identity { + type = "UserAssigned" + identity_ids = [ + azurerm_user_assigned_identity.script_identity.id + ] + } + + environment_variable { + name = "clusterName" + value = var.aks_cluster_name + } + + environment_variable { + name = "resourceGroupName" + value = var.resource_group_name + } + + environment_variable { + name = "applicationGatewayEnabled" + value = false + } + + environment_variable { + name = "tenantId" + value = var.tenant_id + } + + environment_variable { + name = "subscriptionId" + value = var.subscription_id + } + + environment_variable { + name = "hostName" + value = var.hostname + } + + environment_variable { + name = "namespace" + value = var.namespace + } + + environment_variable { + name = "serviceAccountName" + value = var.service_account_name + } + + environment_variable { + name = "workloadManagedIdentityClientId" + value = var.workload_managed_identity_client_id + } + + environment_variable { + name = "email" + value = var.email + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/output.tf new file mode 100644 index 000000000..2b3b8e992 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/output.tf @@ -0,0 +1,9 @@ +output "id" { + value = azurerm_resource_deployment_script_azure_cli.script.id + description = "Specifies the resource id of the deployment script" +} + +output "outputs" { + value = azurerm_resource_deployment_script_azure_cli.script.outputs + description = "Specifies the list of script outputs." +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf new file mode 100644 index 000000000..ca7442247 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf @@ -0,0 +1,78 @@ +variable "resource_group_name" { + description = "(Required) Specifies the resource group name" + type = string +} + +variable "location" { + description = "(Required) Specifies the location of the Azure OpenAI Service" + type = string +} + +variable "name" { + description = "(Required) Specifies the name of the Azure OpenAI Service" + type = string + default = "BashScript" +} + +variable "azure_cli_version" { + description = "(Required) Azure CLI module version to be used." + type = string + default = "2.9.1" +} + +variable "managed_identity_name" { + description = "Specifies the name of the user-defined managed identity used by the deployment script." + type = string + default = "ScriptManagedIdentity" +} + +variable "primary_script_uri" { + description = "(Optional) Uri for the script. This is the entry point for the external script. Changing this forces a new Resource Deployment Script to be created." + type = string +} + +variable "aks_cluster_name" { + description = "Specifies the name of the AKS cluster." + type = string +} + +variable "tenant_id" { + description = "Specifies the Azure AD tenant id." + type = string +} + +variable "subscription_id" { + description = "Specifies the Azure subscription id." + type = string +} + +variable "hostname" { + description = "Specifies the hostname of the application." + type = string +} + +variable "namespace" { + description = "Specifies the namespace of the application." + type = string +} + +variable "service_account_name" { + description = "Specifies the service account of the application." + type = string +} + +variable "workload_managed_identity_client_id" { + description = "Specifies the client id of the workload user-defined managed identity." + type = string +} + +variable "email" { + description = "Specifies the email address for the cert-manager cluster issuer." + type = string +} + +variable "tags" { + description = "(Optional) Specifies the tags of the Azure OpenAI Service" + type = map(any) + default = {} +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf new file mode 100644 index 000000000..4456c789f --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf @@ -0,0 +1,38 @@ +resource "azurerm_monitor_diagnostic_setting" "settings" { + name = var.name + target_resource_id = var.target_resource_id + + log_analytics_workspace_id = var.log_analytics_workspace_id + log_analytics_destination_type = var.log_analytics_destination_type + + eventhub_name = var.eventhub_name + eventhub_authorization_rule_id = var.eventhub_authorization_rule_id + + storage_account_id = var.storage_account_id + + dynamic "log" { + for_each = toset(logs) + content { + category = each.key + enabled = true + + retention_policy { + enabled = var.retention_policy_enabled + days = var.retention_policy_days + } + } + } + + dynamic "metric" { + for_each = toset(metrics) + content { + category = each.key + enabled = true + + retention_policy { + enabled = var.retention_policy_enabled + days = var.retention_policy_days + } + } + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf new file mode 100644 index 000000000..3b15757f8 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf @@ -0,0 +1,9 @@ +output "name" { + value = azurerm_key_vault.key_vault.name + description = "Specifies the name of the key vault." +} + +output "id" { + value = azurerm_key_vault.key_vault.id + description = "Specifies the resource id of the key vault." +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf new file mode 100644 index 000000000..5fefdb86a --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf @@ -0,0 +1,79 @@ + +variable "name" { + description = "(Required) Specifies the name of the Container Registry. Changing this forces a new resource to be created." + type = string +} + +variable "resource_group_name" { + description = "(Required) The name of the resource group in which to create the Container Registry. Changing this forces a new resource to be created." + type = string +} + +variable "location" { + description = "(Required) Specifies the supported Azure location where the resource exists. Changing this forces a new resource to be created." + type = string +} + +variable "retention_policy_enabled" { + description = "(Required) Is this Retention Policy enabled?" + type = bool + default = true +} + +variable "retention_policy_days" { + description = "(Optional) The number of days for which this Retention Policy should apply." + type = number + default = 30 +} + +variable "target_resource_id" { + description = "(Required) The ID of an existing Resource on which to configure Diagnostic Settings. Changing this forces a new resource to be created." + type = string +} + +variable "log_analytics_workspace_id" { + description = "(Optional) Specifies the ID of a Log Analytics Workspace where Diagnostics Data should be sent." + type = string +} + +variable "log_analytics_destination_type" { + description = "(Optional) When set to 'Dedicated' logs sent to a Log Analytics workspace will go into resource specific tables, instead of the legacy AzureDiagnostics table." + type = string + default = null +} + +variable "storage_account_id" { + description = "(Optional) The ID of the Storage Account where logs should be sent. Changing this forces a new resource to be created." + type = string + default = null +} + +variable "eventhub_name" { + description = "(Optional) Specifies the name of the Event Hub where Diagnostics Data should be sent. Changing this forces a new resource to be created." + type = string + default = null +} + +variable "eventhub_authorization_rule_id" { + description = "(Optional) Specifies the ID of an Event Hub Namespace Authorization Rule used to send Diagnostics Data. Changing this forces a new resource to be created." + type = string + default = null +} + +variable "logs" { + description = "(Optional) Specifies a list of log categories to enable." + type = list(string) + default = [] +} + +variable "metrics" { + description = "(Optional) Specifies a list of metrics to enable." + type = list(string) + default = [] +} + +variable "tags" { + description = "(Optional) A mapping of tags to assign to the resource." + type = map(any) + default = {} +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf new file mode 100644 index 000000000..3f535454b --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf @@ -0,0 +1,310 @@ +resource "azurerm_public_ip" "pip" { + name = var.pip_name + resource_group_name = var.resource_group_name + location = var.location + zones = var.zones + allocation_method = "Static" + sku = "Standard" + tags = var.tags + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_firewall" "firewall" { + name = var.name + resource_group_name = var.resource_group_name + location = var.location + zones = var.zones + threat_intel_mode = var.threat_intel_mode + sku_name = var.sku_name + sku_tier = var.sku_tier + firewall_policy_id = azurerm_firewall_policy.policy.id + tags = var.tags + + + ip_configuration { + name = "fw_ip_config" + subnet_id = var.subnet_id + public_ip_address_id = azurerm_public_ip.pip.id + } + + lifecycle { + ignore_changes = [ + tags, + + ] + } +} + +resource "azurerm_firewall_policy" "policy" { + name = "${var.name}Policy" + resource_group_name = var.resource_group_name + location = var.location + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_firewall_policy_rule_collection_group" "policy" { + name = "AksEgressPolicyRuleCollectionGroup" + firewall_policy_id = azurerm_firewall_policy.policy.id + priority = 500 + + application_rule_collection { + name = "ApplicationRules" + priority = 500 + action = "Allow" + + rule { + name = "AllowMicrosoftFqdns" + source_addresses = ["*"] + + destination_fqdns = [ + "*.cdn.mscr.io", + "mcr.microsoft.com", + "*.data.mcr.microsoft.com", + "management.azure.com", + "login.microsoftonline.com", + "acs-mirror.azureedge.net", + "dc.services.visualstudio.com", + "*.opinsights.azure.com", + "*.oms.opinsights.azure.com", + "*.microsoftonline.com", + "*.monitoring.azure.com", + ] + + protocols { + port = "80" + type = "Http" + } + + protocols { + port = "443" + type = "Https" + } + } + + rule { + name = "AllowFqdnsForOsUpdates" + source_addresses = ["*"] + + destination_fqdns = [ + "download.opensuse.org", + "security.ubuntu.com", + "ntp.ubuntu.com", + "packages.microsoft.com", + "snapcraft.io" + ] + + protocols { + port = "80" + type = "Http" + } + + protocols { + port = "443" + type = "Https" + } + } + + rule { + name = "AllowImagesFqdns" + source_addresses = ["*"] + + destination_fqdns = [ + "auth.docker.io", + "registry-1.docker.io", + "production.cloudflare.docker.com" + ] + + protocols { + port = "80" + type = "Http" + } + + protocols { + port = "443" + type = "Https" + } + } + + rule { + name = "AllowBing" + source_addresses = ["*"] + + destination_fqdns = [ + "*.bing.com" + ] + + protocols { + port = "80" + type = "Http" + } + + protocols { + port = "443" + type = "Https" + } + } + + rule { + name = "AllowGoogle" + source_addresses = ["*"] + + destination_fqdns = [ + "*.google.com" + ] + + protocols { + port = "80" + type = "Http" + } + + protocols { + port = "443" + type = "Https" + } + } + } + + network_rule_collection { + name = "NetworkRules" + priority = 400 + action = "Allow" + + rule { + name = "Time" + source_addresses = ["*"] + destination_ports = ["123"] + destination_addresses = ["*"] + protocols = ["UDP"] + } + + rule { + name = "DNS" + source_addresses = ["*"] + destination_ports = ["53"] + destination_addresses = ["*"] + protocols = ["UDP"] + } + + rule { + name = "ServiceTags" + source_addresses = ["*"] + destination_ports = ["*"] + destination_addresses = [ + "AzureContainerRegistry", + "MicrosoftContainerRegistry", + "AzureActiveDirectory" + ] + protocols = ["Any"] + } + + rule { + name = "Internet" + source_addresses = ["*"] + destination_ports = ["*"] + destination_addresses = ["*"] + protocols = ["TCP"] + } + } + + lifecycle { + ignore_changes = [ + application_rule_collection, + network_rule_collection, + nat_rule_collection + ] + } +} + +resource "azurerm_monitor_diagnostic_setting" "settings" { + name = "DiagnosticsSettings" + target_resource_id = azurerm_firewall.firewall.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "AzureFirewallApplicationRule" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "AzureFirewallNetworkRule" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "AzureFirewallDnsProxy" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + metric { + category = "AllMetrics" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } +} + +resource "azurerm_monitor_diagnostic_setting" "pip_settings" { + name = "DiagnosticsSettings" + target_resource_id = azurerm_public_ip.pip.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "DDoSProtectionNotifications" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "DDoSMitigationFlowLogs" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "DDoSMitigationReports" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + metric { + category = "AllMetrics" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } +} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf new file mode 100644 index 000000000..b11aab5ea --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf @@ -0,0 +1,4 @@ +output "private_ip_address" { + description = "Specifies the private IP address of the firewall." + value = azurerm_firewall.firewall.ip_configuration[0].private_ip_address +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf new file mode 100644 index 000000000..dedd9481b --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf @@ -0,0 +1,80 @@ +variable "name" { + description = "Specifies the firewall name" + type = string +} + +variable "sku_name" { + description = "(Required) SKU name of the Firewall. Possible values are AZFW_Hub and AZFW_VNet. Changing this forces a new resource to be created." + default = "AZFW_VNet" + type = string + + validation { + condition = contains(["AZFW_Hub", "AZFW_VNet" ], var.sku_name) + error_message = "The value of the sku name property of the firewall is invalid." + } +} + +variable "sku_tier" { + description = "(Required) SKU tier of the Firewall. Possible values are Premium, Standard, and Basic." + default = "Standard" + type = string + + validation { + condition = contains(["Premium", "Standard", "Basic" ], var.sku_tier) + error_message = "The value of the sku tier property of the firewall is invalid." + } +} + +variable "resource_group_name" { + description = "Specifies the resource group name" + type = string +} + +variable "location" { + description = "Specifies the location where firewall will be deployed" + type = string +} + +variable "threat_intel_mode" { + description = "(Optional) The operation mode for threat intelligence-based filtering. Possible values are: Off, Alert, Deny. Defaults to Alert." + default = "Alert" + type = string + + validation { + condition = contains(["Off", "Alert", "Deny"], var.threat_intel_mode) + error_message = "The threat intel mode is invalid." + } +} + +variable "zones" { + description = "Specifies the availability zones of the Azure Firewall" + default = ["1", "2", "3"] + type = list(string) +} + +variable "pip_name" { + description = "Specifies the firewall public IP name" + type = string + default = "azure-fw-ip" +} + +variable "subnet_id" { + description = "Subnet ID" + type = string +} + +variable "tags" { + description = "(Optional) Specifies the tags of the storage account" + default = {} +} + +variable "log_analytics_workspace_id" { + description = "Specifies the log analytics workspace id" + type = string +} + +variable "log_analytics_retention_days" { + description = "Specifies the number of days of the retention policy" + type = number + default = 7 +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf new file mode 100644 index 000000000..df166f775 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf @@ -0,0 +1,64 @@ +resource "azurerm_key_vault" "key_vault" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + tenant_id = var.tenant_id + sku_name = var.sku_name + tags = var.tags + enabled_for_deployment = var.enabled_for_deployment + enabled_for_disk_encryption = var.enabled_for_disk_encryption + enabled_for_template_deployment = var.enabled_for_template_deployment + enable_rbac_authorization = var.enable_rbac_authorization + purge_protection_enabled = var.purge_protection_enabled + soft_delete_retention_days = var.soft_delete_retention_days + + timeouts { + delete = "60m" + } + + network_acls { + bypass = var.bypass + default_action = var.default_action + ip_rules = var.ip_rules + virtual_network_subnet_ids = var.virtual_network_subnet_ids + } + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_monitor_diagnostic_setting" "settings" { + name = "DiagnosticsSettings" + target_resource_id = azurerm_key_vault.key_vault.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "AuditEvent" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "AzurePolicyEvaluationDetails" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + metric { + category = "AllMetrics" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf new file mode 100644 index 000000000..3b15757f8 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf @@ -0,0 +1,9 @@ +output "name" { + value = azurerm_key_vault.key_vault.name + description = "Specifies the name of the key vault." +} + +output "id" { + value = azurerm_key_vault.key_vault.id + description = "Specifies the resource id of the key vault." +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf new file mode 100644 index 000000000..df4cdbe55 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf @@ -0,0 +1,115 @@ +variable "name" { + description = "(Required) Specifies the name of the key vault." + type = string +} + +variable "resource_group_name" { + description = "(Required) Specifies the resource group name of the key vault." + type = string +} + +variable "location" { + description = "(Required) Specifies the location where the key vault will be deployed." + type = string +} + +variable "tenant_id" { + description = "(Required) The Azure Active Directory tenant ID that should be used for authenticating requests to the key vault." + type = string +} + +variable "sku_name" { + description = "(Required) The Name of the SKU used for this Key Vault. Possible values are standard and premium." + type = string + default = "standard" + + validation { + condition = contains(["standard", "premium" ], var.sku_name) + error_message = "The value of the sku name property of the key vault is invalid." + } +} + +variable "tags" { + description = "(Optional) Specifies the tags of the log analytics workspace" + type = map(any) + default = {} +} + +variable "enabled_for_deployment" { + description = "(Optional) Boolean flag to specify whether Azure Virtual Machines are permitted to retrieve certificates stored as secrets from the key vault. Defaults to false." + type = bool + default = false +} + +variable "enabled_for_disk_encryption" { + description = " (Optional) Boolean flag to specify whether Azure Disk Encryption is permitted to retrieve secrets from the vault and unwrap keys. Defaults to false." + type = bool + default = false +} + +variable "enabled_for_template_deployment" { + description = "(Optional) Boolean flag to specify whether Azure Resource Manager is permitted to retrieve secrets from the key vault. Defaults to false." + type = bool + default = false +} + +variable "enable_rbac_authorization" { + description = "(Optional) Boolean flag to specify whether Azure Key Vault uses Role Based Access Control (RBAC) for authorization of data actions. Defaults to false." + type = bool + default = false +} + +variable "purge_protection_enabled" { + description = "(Optional) Is Purge Protection enabled for this Key Vault? Defaults to false." + type = bool + default = false +} + +variable "soft_delete_retention_days" { + description = "(Optional) The number of days that items should be retained for once soft-deleted. This value can be between 7 and 90 (the default) days." + type = number + default = 30 +} + +variable "bypass" { + description = "(Required) Specifies which traffic can bypass the network rules. Possible values are AzureServices and None." + type = string + default = "AzureServices" + + validation { + condition = contains(["AzureServices", "None" ], var.bypass) + error_message = "The valut of the bypass property of the key vault is invalid." + } +} + +variable "default_action" { + description = "(Required) The Default Action to use when no rules match from ip_rules / virtual_network_subnet_ids. Possible values are Allow and Deny." + type = string + default = "Allow" + + validation { + condition = contains(["Allow", "Deny" ], var.default_action) + error_message = "The value of the default action property of the key vault is invalid." + } +} + +variable "ip_rules" { + description = "(Optional) One or more IP Addresses, or CIDR Blocks which should be able to access the Key Vault." + default = [] +} + +variable "virtual_network_subnet_ids" { + description = "(Optional) One or more Subnet ID's which should be able to access this Key Vault." + default = [] +} + +variable "log_analytics_workspace_id" { + description = "Specifies the log analytics workspace id" + type = string +} + +variable "log_analytics_retention_days" { + description = "Specifies the number of days of the retention policy" + type = number + default = 7 +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf new file mode 100644 index 000000000..fc3a1d85a --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf @@ -0,0 +1,35 @@ +resource "azurerm_log_analytics_workspace" "log_analytics_workspace" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + sku = var.sku + tags = var.tags + retention_in_days = var.retention_in_days != "" ? var.retention_in_days : null + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_log_analytics_solution" "la_solution" { + for_each = var.solution_plan_map + + solution_name = each.key + location = var.location + resource_group_name = var.resource_group_name + workspace_resource_id = azurerm_log_analytics_workspace.log_analytics_workspace.id + workspace_name = azurerm_log_analytics_workspace.log_analytics_workspace.name + + plan { + product = each.value.product + publisher = each.value.publisher + } + + lifecycle { + ignore_changes = [ + tags + ] + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf new file mode 100644 index 000000000..8cb42544a --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf @@ -0,0 +1,30 @@ +output "id" { + value = azurerm_log_analytics_workspace.log_analytics_workspace.id + description = "Specifies the resource id of the log analytics workspace" +} + +output "location" { + value = azurerm_log_analytics_workspace.log_analytics_workspace.location + description = "Specifies the location of the log analytics workspace" +} + +output "name" { + value = azurerm_log_analytics_workspace.log_analytics_workspace.name + description = "Specifies the name of the log analytics workspace" +} + +output "resource_group_name" { + value = azurerm_log_analytics_workspace.log_analytics_workspace.resource_group_name + description = "Specifies the name of the resource group that contains the log analytics workspace" +} + +output "workspace_id" { + value = azurerm_log_analytics_workspace.log_analytics_workspace.workspace_id + description = "Specifies the workspace id of the log analytics workspace" +} + +output "primary_shared_key" { + value = azurerm_log_analytics_workspace.log_analytics_workspace.primary_shared_key + description = "Specifies the workspace key of the log analytics workspace" + sensitive = true +} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf new file mode 100644 index 000000000..107a0a8da --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf @@ -0,0 +1,43 @@ +variable "resource_group_name" { + description = "(Required) Specifies the resource group name" + type = string +} + +variable "location" { + description = "(Required) Specifies the location of the log analytics workspace" + type = string +} + +variable "name" { + description = "(Required) Specifies the name of the log analytics workspace" + type = string +} + +variable "sku" { + description = "(Optional) Specifies the sku of the log analytics workspace" + type = string + default = "PerGB2018" + + validation { + condition = contains(["Free", "Standalone", "PerNode", "PerGB2018"], var.sku) + error_message = "The log analytics sku is incorrect." + } +} + +variable "solution_plan_map" { + description = "(Optional) Specifies the map structure containing the list of solutions to be enabled." + type = map(any) + default = {} +} + +variable "tags" { + description = "(Optional) Specifies the tags of the log analytics workspace" + type = map(any) + default = {} +} + +variable "retention_in_days" { + description = " (Optional) Specifies the workspace data retention in days. Possible values are either 7 (Free Tier only) or range between 30 and 730." + type = number + default = 30 +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf new file mode 100644 index 000000000..74e201a8c --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf @@ -0,0 +1,42 @@ +resource "azurerm_public_ip" "nat_gategay_public_ip" { + name = "${var.name}PublicIp" + location = var.location + resource_group_name = var.resource_group_name + allocation_method = "Static" + sku = "Standard" + zones = var.zones + tags = var.tags + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_nat_gateway" "nat_gateway" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + sku_name = var.sku_name + idle_timeout_in_minutes = var.idle_timeout_in_minutes + zones = var.zones + tags = var.tags + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_association" { + nat_gateway_id = azurerm_nat_gateway.nat_gateway.id + public_ip_address_id = azurerm_public_ip.nat_gategay_public_ip.id +} + +resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { + for_each = var.subnet_ids + subnet_id = each.value + nat_gateway_id = azurerm_nat_gateway.nat_gateway.id +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf new file mode 100644 index 000000000..014ece6b0 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf @@ -0,0 +1,14 @@ +output "name" { + value = azurerm_nat_gateway.nat_gateway.name + description = "Specifies the name of the Azure NAT Gateway" +} + +output "id" { + value = azurerm_nat_gateway.nat_gateway.id + description = "Specifies the resource id of the Azure NAT Gateway" +} + +output "public_ip_address" { + value = azurerm_public_ip.nat_gategay_public_ip.ip_address + description = "Contains the public IP address of the Azure NAT Gateway." +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf new file mode 100644 index 000000000..0e11ddadc --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf @@ -0,0 +1,43 @@ +variable "resource_group_name" { + description = "(Required) Specifies the resource group name" + type = string +} + +variable "location" { + description = "(Required) Specifies the location of the Azure OpenAI Service" + type = string +} + +variable "name" { + description = "(Required) Specifies the name of the Azure OpenAI Service" + type = string +} + +variable "tags" { + description = "(Optional) Specifies the tags of the Azure OpenAI Service" + type = map(any) + default = {} +} + +variable "sku_name" { + description = "(Optional) The SKU which should be used. At this time the only supported value is Standard. Defaults to Standard" + type = string + default = "Standard" +} + +variable "idle_timeout_in_minutes" { + description = "(Optional) The idle timeout which should be used in minutes. Defaults to 4." + type = number + default = 4 +} + +variable "zones" { + description = " (Optional) A list of Availability Zones in which this NAT Gateway should be located. Changing this forces a new NAT Gateway to be created." + type = list(string) + default = [] +} + +variable "subnet_ids" { + description = "(Required) A map of subnet ids to associate with the NAT Gateway" + type = map(string) +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf new file mode 100644 index 000000000..80edbd556 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf @@ -0,0 +1,58 @@ +resource "azurerm_network_security_group" "nsg" { + name = var.name + resource_group_name = var.resource_group_name + location = var.location + tags = var.tags + + dynamic "security_rule" { + for_each = try(var.security_rules, []) + content { + name = try(security_rule.value.name, null) + priority = try(security_rule.value.priority, null) + direction = try(security_rule.value.direction, null) + access = try(security_rule.value.access, null) + protocol = try(security_rule.value.protocol, null) + source_port_range = try(security_rule.value.source_port_range, null) + source_port_ranges = try(security_rule.value.source_port_ranges, null) + destination_port_range = try(security_rule.value.destination_port_range, null) + destination_port_ranges = try(security_rule.value.destination_port_ranges, null) + source_address_prefix = try(security_rule.value.source_address_prefix, null) + source_address_prefixes = try(security_rule.value.source_address_prefixes, null) + destination_address_prefix = try(security_rule.value.destination_address_prefix, null) + destination_address_prefixes = try(security_rule.value.destination_address_prefixes, null) + source_application_security_group_ids = try(security_rule.value.source_application_security_group_ids, null) + destination_application_security_group_ids = try(security_rule.value.destination_application_security_group_ids, null) + } + } + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_monitor_diagnostic_setting" "settings" { + name = "DiagnosticsSettings" + target_resource_id = azurerm_network_security_group.nsg.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "NetworkSecurityGroupEvent" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "NetworkSecurityGroupRuleCounter" + enabled = true + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/outputs.tf new file mode 100644 index 000000000..ca2a13e32 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/outputs.tf @@ -0,0 +1,4 @@ +output "id" { + description = "Specifies the resource id of the network security group" + value = azurerm_network_security_group.nsg.id +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/variables.tf new file mode 100644 index 000000000..1de3c61ad --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/variables.tf @@ -0,0 +1,36 @@ +variable "name" { + description = "(Required) Specifies the name of the network security group" + type = string +} + +variable "resource_group_name" { + description = "(Required) Specifies the resource group name of the network security group" + type = string +} + +variable "location" { + description = "(Required) Specifies the location of the network security group" + type = string +} + +variable "security_rules" { + description = "(Optional) Specifies the security rules of the network security group" + type = list(object) + default = [] +} + +variable "tags" { + description = "(Optional) Specifies the tags of the network security group" + default = {} +} + +variable "log_analytics_workspace_id" { + description = "Specifies the log analytics workspace resource id" + type = string +} + +variable "log_analytics_retention_days" { + description = "Specifies the number of days of the retention policy" + type = number + default = 7 +} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf new file mode 100644 index 000000000..e13f1340b --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf @@ -0,0 +1,31 @@ +resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { + kubernetes_cluster_id = var.kubernetes_cluster_id + name = var.name + vm_size = var.vm_size + mode = var.mode + node_labels = var.node_labels + node_taints = var.node_taints + zones = var.availability_zones + vnet_subnet_id = var.vnet_subnet_id + pod_subnet_id = var.pod_subnet_id + enable_auto_scaling = var.enable_auto_scaling + enable_host_encryption = var.enable_host_encryption + enable_node_public_ip = var.enable_node_public_ip + proximity_placement_group_id = var.proximity_placement_group_id + orchestrator_version = var.orchestrator_version + max_pods = var.max_pods + max_count = var.max_count + min_count = var.min_count + node_count = var.node_count + os_disk_size_gb = var.os_disk_size_gb + os_disk_type = var.os_disk_type + os_type = var.os_type + priority = var.priority + tags = var.tags + + lifecycle { + ignore_changes = [ + tags + ] + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/outputs.tf new file mode 100644 index 000000000..936f87b5c --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/outputs.tf @@ -0,0 +1,4 @@ +output "id" { + description = "Specifies the resource id of the node pool" + value = azurerm_kubernetes_cluster_node_pool.node_pool.id +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf new file mode 100644 index 000000000..688b179b8 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf @@ -0,0 +1,144 @@ +variable "name" { + description = "(Required) Specifies the name of the node pool." + type = string +} + +variable "kubernetes_cluster_id" { + description = "(Required) Specifies the resource id of the AKS cluster." + type = string +} + +variable "vm_size" { + description = "(Required) The SKU which should be used for the Virtual Machines used in this Node Pool. Changing this forces a new resource to be created." + type = string +} + +variable "availability_zones" { + description = "(Optional) A list of Availability Zones where the Nodes in this Node Pool should be created in. Changing this forces a new resource to be created." + type = list(string) + default = ["1", "2", "3"] +} + +variable "enable_auto_scaling" { + description = "(Optional) Whether to enable auto-scaler. Defaults to false." + type = bool + default = false +} + +variable "enable_host_encryption" { + description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." + type = bool + default = false +} + +variable "enable_node_public_ip" { + description = "(Optional) Should each node have a Public IP Address? Defaults to false. Changing this forces a new resource to be created." + type = bool + default = false +} + +variable "max_pods" { + description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." + type = number + default = 250 +} + +variable "mode" { + description = "(Optional) Should this Node Pool be used for System or User resources? Possible values are System and User. Defaults to User." + type = string + default = "User" +} + +variable "node_labels" { + description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." + type = map(any) + default = {} +} + +variable "node_taints" { + description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." + type = list(string) + default = [] +} + +variable "tags" { + description = "(Optional) Specifies the tags of the network security group" + default = {} +} + +variable "orchestrator_version" { + description = "(Optional) Version of Kubernetes used for the Agents. If not specified, the latest recommended version will be used at provisioning time (but won't auto-upgrade)" + type = string + default = null +} + +variable "os_disk_size_gb" { + description = "(Optional) The Agent Operating System disk size in GB. Changing this forces a new resource to be created." + type = number + default = null +} + +variable "os_disk_type" { + description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." + type = string + default = "Ephemeral" +} + +variable "os_type" { + description = "(Optional) The Operating System which should be used for this Node Pool. Changing this forces a new resource to be created. Possible values are Linux and Windows. Defaults to Linux." + type = string + default = "Linux" +} + +variable "priority" { + description = "(Optional) The Priority for Virtual Machines within the Virtual Machine Scale Set that powers this Node Pool. Possible values are Regular and Spot. Defaults to Regular. Changing this forces a new resource to be created." + type = string + default = "Regular" +} + +variable "proximity_placement_group_id" { + description = "(Optional) The ID of the Proximity Placement Group where the Virtual Machine Scale Set that powers this Node Pool will be placed. Changing this forces a new resource to be created." + type = string + default = null +} + +variable "vnet_subnet_id" { + description = "(Optional) The ID of the Subnet where this Node Pool should exist." + type = string + default = null +} + +variable "pod_subnet_id" { + description = "(Optional) The ID of the Subnet where the pods in the system node pool should exist. Changing this forces a new resource to be created." + type = string + default = null +} + +variable "max_count" { + description = "(Required) The maximum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be greater than or equal to min_count." + type = number + default = 10 +} + +variable "min_count" { + description = "(Required) The minimum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be less than or equal to max_count." + type = number + default = 3 +} + +variable "node_count" { + description = "(Optional) The initial number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be a value in the range min_count - max_count." + type = number + default = 3 +} + +variable resource_group_name { + description = "Specifies the resource group name" + type = string +} + +variable "oidc_issuer_enabled" { + description = " (Optional) Enable or Disable the OIDC issuer URL." + type = bool + default = true +} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf new file mode 100644 index 000000000..8af163a57 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf @@ -0,0 +1,79 @@ +resource "azurerm_cognitive_account" "openai" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + kind = "OpenAI" + custom_subdomain_name = var.custom_subdomain_name + sku_name = var.sku_name + public_network_access_enabled = var.public_network_access_enabled + tags = var.tags + + identity { + type = "SystemAssigned" + } + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_cognitive_deployment" "deployment" { + for_each = {for deployment in var.deployments: deployment.name => deployment} + + name = each.key + cognitive_account_id = azurerm_cognitive_account.openai.id + + model { + format = "OpenAI" + name = each.value.model.name + version = each.value.model.version + } + + scale { + type = "Standard" + } +} + +resource "azurerm_monitor_diagnostic_setting" "settings" { + name = "DiagnosticsSettings" + target_resource_id = azurerm_cognitive_account.openai.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "Audit" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "RequestResponse" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + enabled_log { + category = "Trace" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + metric { + category = "AllMetrics" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf new file mode 100644 index 000000000..85097ba3d --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf @@ -0,0 +1,34 @@ +output "id" { + value = azurerm_cognitive_account.openai.id + description = "Specifies the resource id of the log analytics workspace" +} + +output "location" { + value = azurerm_cognitive_account.openai.location + description = "Specifies the location of the log analytics workspace" +} + +output "name" { + value = azurerm_cognitive_account.openai.name + description = "Specifies the name of the log analytics workspace" +} + +output "resource_group_name" { + value = azurerm_cognitive_account.openai.resource_group_name + description = "Specifies the name of the resource group that contains the log analytics workspace" +} + +output "endpoint" { + value = azurerm_cognitive_account.openai.endpoint + description = "Specifies the endpoint of the Azure OpenAI Service." +} + +output "primary_access_key" { + value = azurerm_cognitive_account.openai.endpoint + description = "Specifies the primary access key of the Azure OpenAI Service." +} + +output "secondary_access_key" { + value = azurerm_cognitive_account.openai.endpoint + description = "Specifies the secondary access key of the Azure OpenAI Service." +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf new file mode 100644 index 000000000..1d13d78a6 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf @@ -0,0 +1,70 @@ +variable "resource_group_name" { + description = "(Required) Specifies the resource group name" + type = string +} + +variable "location" { + description = "(Required) Specifies the location of the Azure OpenAI Service" + type = string +} + +variable "name" { + description = "(Required) Specifies the name of the Azure OpenAI Service" + type = string +} + +variable "sku_name" { + description = "(Optional) Specifies the sku name for the Azure OpenAI Service" + type = string + default = "S0" +} + +variable "tags" { + description = "(Optional) Specifies the tags of the Azure OpenAI Service" + type = map(any) + default = {} +} + +variable "custom_subdomain_name" { + description = "(Optional) Specifies the custom subdomain name of the Azure OpenAI Service" + type = string +} + +variable "public_network_access_enabled" { + description = "(Optional) Specifies whether public network access is allowed for the Azure OpenAI Service" + type = bool + default = true +} + +variable "deployments" { + description = "(Optional) Specifies the deployments of the Azure OpenAI Service" + type = list(object({ + name = string + model = object({ + name = string + version = string + }) + rai_policy_name = string + })) + default = [ + { + name = "gpt-35-turbo" + model = { + name = "gpt-35-turbo" + version = "0301" + } + rai_policy_name = "" + } + ] +} + +variable "log_analytics_workspace_id" { + description = "Specifies the log analytics workspace id" + type = string +} + +variable "log_analytics_retention_days" { + description = "Specifies the number of days of the retention policy" + type = number + default = 7 +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf new file mode 100644 index 000000000..fb97cc407 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf @@ -0,0 +1,26 @@ +resource "azurerm_private_dns_zone" "private_dns_zone" { + name = var.name + resource_group_name = var.resource_group_name + tags = var.tags + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_private_dns_zone_virtual_network_link" "link" { + for_each = var.virtual_networks_to_link + + name = "link_to_${lower(basename(each.key))}" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.private_dns_zone.name + virtual_network_id = "/subscriptions/${each.value.subscription_id}/resourceGroups/${each.value.resource_group_name}/providers/Microsoft.Network/virtualNetworks/${each.key}" + + lifecycle { + ignore_changes = [ + tags + ] + } +} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/outputs.tf new file mode 100644 index 000000000..c37a77f92 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/outputs.tf @@ -0,0 +1,4 @@ +output "id" { + description = "Specifies the resource id of the private dns zone" + value = azurerm_private_dns_zone.private_dns_zone.id +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf new file mode 100644 index 000000000..b687d39cd --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf @@ -0,0 +1,20 @@ +variable "name" { + description = "(Required) Specifies the name of the private dns zone" + type = string +} + +variable "resource_group_name" { + description = "(Required) Specifies the resource group name of the private dns zone" + type = string +} + +variable "tags" { + description = "(Optional) Specifies the tags of the private dns zone" + default = {} +} + +variable "virtual_networks_to_link" { + description = "(Optional) Specifies the subscription id, resource group name, and name of the virtual networks to which create a virtual network link" + type = map(any) + default = {} +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf new file mode 100644 index 000000000..ae49a166e --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf @@ -0,0 +1,26 @@ +resource "azurerm_private_endpoint" "private_endpoint" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.subnet_id + tags = var.tags + + private_service_connection { + name = "${var.name}Connection" + private_connection_resource_id = var.private_connection_resource_id + is_manual_connection = var.is_manual_connection + subresource_names = try([var.subresource_name], null) + request_message = try(var.request_message, null) + } + + private_dns_zone_group { + name = var.private_dns_zone_group_name + private_dns_zone_ids = var.private_dns_zone_group_ids + } + + lifecycle { + ignore_changes = [ + tags + ] + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/outputs.tf new file mode 100644 index 000000000..ef51964b0 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/outputs.tf @@ -0,0 +1,14 @@ +output "id" { + description = "Specifies the resource id of the private endpoint." + value = azurerm_private_endpoint.private_endpoint.id +} + +output "private_dns_zone_group" { + description = "Specifies the private dns zone group of the private endpoint." + value = azurerm_private_endpoint.private_endpoint.private_dns_zone_group +} + +output "private_dns_zone_configs" { + description = "Specifies the private dns zone(s) configuration" + value = azurerm_private_endpoint.private_endpoint.private_dns_zone_configs +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf new file mode 100644 index 000000000..5d9c44048 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf @@ -0,0 +1,61 @@ +variable "name" { + description = "(Required) Specifies the name of the private endpoint. Changing this forces a new resource to be created." + type = string +} + +variable "resource_group_name" { + description = "(Required) The name of the resource group. Changing this forces a new resource to be created." + type = string +} + +variable "private_connection_resource_id" { + description = "(Required) Specifies the resource id of the private link service" + type = string +} + +variable "location" { + description = "(Required) Specifies the supported Azure location where the resource exists. Changing this forces a new resource to be created." + type = string +} + +variable "subnet_id" { + description = "(Required) Specifies the resource id of the subnet" + type = string +} + +variable "is_manual_connection" { + description = "(Optional) Specifies whether the private endpoint connection requires manual approval from the remote resource owner." + type = string + default = false +} + +variable "subresource_name" { + description = "(Optional) Specifies a subresource name which the Private Endpoint is able to connect to." + type = string + default = null +} + +variable "request_message" { + description = "(Optional) Specifies a message passed to the owner of the remote resource when the private endpoint attempts to establish the connection to the remote resource." + type = string + default = null +} + +variable "private_dns_zone_group_name" { + description = "(Required) Specifies the Name of the Private DNS Zone Group. Changing this forces a new private_dns_zone_group resource to be created." + type = string +} + +variable "private_dns_zone_group_ids" { + description = "(Required) Specifies the list of Private DNS Zones to include within the private_dns_zone_group." + type = list(string) +} + +variable "tags" { + description = "(Optional) Specifies the tags of the network security group" + default = {} +} + +variable "private_dns" { + default = {} +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf new file mode 100644 index 000000000..0f9a4b649 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf @@ -0,0 +1,30 @@ +data "azurerm_client_config" "current" { +} + +resource "azurerm_route_table" "rt" { + name = var.route_table_name + location = var.location + resource_group_name = var.resource_group_name + tags = var.tags + + route { + name = "kubenetfw_fw_r" + address_prefix = "0.0.0.0/0" + next_hop_type = "VirtualAppliance" + next_hop_in_ip_address = var.firewall_private_ip + } + + lifecycle { + ignore_changes = [ + tags, + route + ] + } +} + +resource "azurerm_subnet_route_table_association" "subnet_association" { + for_each = var.subnets_to_associate + + subnet_id = "/subscriptions/${each.value.subscription_id}/resourceGroups/${each.value.resource_group_name}/providers/Microsoft.Network/virtualNetworks/${each.value.virtual_network_name}/subnets/${each.key}" + route_table_id = azurerm_route_table.rt.id +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/route_table/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/route_table/variables.tf new file mode 100644 index 000000000..6102e8065 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/route_table/variables.tf @@ -0,0 +1,35 @@ +variable "resource_group_name" { + description = "Resource group where RouteTable will be deployed" + type = string +} + +variable "location" { + description = "Location where RouteTable will be deployed" + type = string +} + +variable "route_table_name" { + description = "RouteTable name" + type = string +} + +variable "route_name" { + description = "AKS route name" + type = string +} + +variable "firewall_private_ip" { + description = "Firewall private IP" + type = string +} + +variable "subnets_to_associate" { + description = "(Optional) Specifies the subscription id, resource group name, and name of the subnets to associate" + type = map(any) + default = {} +} + +variable "tags" { + description = "(Optional) Specifies the tags of the storage account" + default = {} +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf new file mode 100644 index 000000000..fdaccb7bd --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf @@ -0,0 +1,27 @@ +resource "azurerm_storage_account" "storage_account" { + name = var.name + resource_group_name = var.resource_group_name + + location = var.location + account_kind = var.account_kind + account_tier = var.account_tier + account_replication_type = var.replication_type + is_hns_enabled = var.is_hns_enabled + tags = var.tags + + network_rules { + default_action = (length(var.ip_rules) + length(var.virtual_network_subnet_ids)) > 0 ? "Deny" : var.default_action + ip_rules = var.ip_rules + virtual_network_subnet_ids = var.virtual_network_subnet_ids + } + + identity { + type = "SystemAssigned" + } + + lifecycle { + ignore_changes = [ + tags + ] + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf new file mode 100644 index 000000000..c61fdd254 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf @@ -0,0 +1,24 @@ +output "name" { + description = "Specifies the name of the storage account" + value = azurerm_storage_account.storage_account.name +} + +output "id" { + description = "Specifies the resource id of the storage account" + value = azurerm_storage_account.storage_account.id +} + +output "primary_access_key" { + description = "Specifies the primary access key of the storage account" + value = azurerm_storage_account.storage_account.primary_access_key +} + +output "principal_id" { + description = "Specifies the principal id of the system assigned managed identity of the storage account" + value = azurerm_storage_account.storage_account.identity[0].principal_id +} + +output "primary_blob_endpoint" { + description = "Specifies the primary blob endpoint of the storage account" + value = azurerm_storage_account.storage_account.primary_blob_endpoint +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf new file mode 100644 index 000000000..5122b841c --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf @@ -0,0 +1,81 @@ +variable "resource_group_name" { + description = "(Required) Specifies the resource group name of the storage account" + type = string +} + +variable "name" { + description = "(Required) Specifies the name of the storage account" + type = string +} + +variable "location" { + description = "(Required) Specifies the location of the storage account" + type = string +} + +variable "account_kind" { + description = "(Optional) Specifies the account kind of the storage account" + default = "StorageV2" + type = string + + validation { + condition = contains(["Storage", "StorageV2"], var.account_kind) + error_message = "The account kind of the storage account is invalid." + } +} + +variable "account_tier" { + description = "(Optional) Specifies the account tier of the storage account" + default = "Standard" + type = string + + validation { + condition = contains(["Standard", "Premium"], var.account_tier) + error_message = "The account tier of the storage account is invalid." + } +} + +variable "replication_type" { + description = "(Optional) Specifies the replication type of the storage account" + default = "LRS" + type = string + + validation { + condition = contains(["LRS", "ZRS", "GRS", "GZRS", "RA-GRS", "RA-GZRS"], var.replication_type) + error_message = "The replication type of the storage account is invalid." + } +} + +variable "is_hns_enabled" { + description = "(Optional) Specifies the replication type of the storage account" + default = false + type = bool +} + +variable "default_action" { + description = "Allow or disallow public access to all blobs or containers in the storage accounts. The default interpretation is true for this property." + default = "Allow" + type = string +} + +variable "ip_rules" { + description = "Specifies IP rules for the storage account" + default = [] + type = list(string) +} + +variable "virtual_network_subnet_ids" { + description = "Specifies a list of resource ids for subnets" + default = [] + type = list(string) +} + +variable "kind" { + description = "(Optional) Specifies the kind of the storage account" + default = "" +} + +variable "tags" { + description = "(Optional) Specifies the tags of the storage account" + default = {} +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/main.tf new file mode 100644 index 000000000..b5169c64c --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/main.tf @@ -0,0 +1,221 @@ +resource "azurerm_public_ip" "public_ip" { + name = "${var.name}PublicIp" + location = var.location + resource_group_name = var.resource_group_name + allocation_method = "Dynamic" + domain_name_label = lower(var.name) + count = var.public_ip ? 1 : 0 + tags = var.tags + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_network_security_group" "nsg" { + name = "${var.name}Nsg" + location = var.location + resource_group_name = var.resource_group_name + tags = var.tags + + security_rule { + name = "SSH" + priority = 1001 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "*" + destination_address_prefix = "*" + } + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_network_interface" "nic" { + name = "${var.name}Nic" + location = var.location + resource_group_name = var.resource_group_name + tags = var.tags + + ip_configuration { + name = "Configuration" + subnet_id = var.subnet_id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = try(azurerm_public_ip.public_ip[0].id, null) + } + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_network_interface_security_group_association" "nsg_association" { + network_interface_id = azurerm_network_interface.nic.id + network_security_group_id = azurerm_network_security_group.nsg.id + depends_on = [azurerm_network_security_group.nsg] +} + +resource "azurerm_linux_virtual_machine" "virtual_machine" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + network_interface_ids = [azurerm_network_interface.nic.id] + size = var.size + computer_name = var.name + admin_username = var.vm_user + tags = var.tags + + os_disk { + name = "${var.name}OsDisk" + caching = "ReadWrite" + storage_account_type = var.os_disk_storage_account_type + } + + admin_ssh_key { + username = var.vm_user + public_key = var.admin_ssh_public_key + } + + source_image_reference { + offer = lookup(var.os_disk_image, "offer", null) + publisher = lookup(var.os_disk_image, "publisher", null) + sku = lookup(var.os_disk_image, "sku", null) + version = lookup(var.os_disk_image, "version", null) + } + + boot_diagnostics { + storage_account_uri = var.boot_diagnostics_storage_account == "" ? null : var.boot_diagnostics_storage_account + } + + lifecycle { + ignore_changes = [ + tags + ] + } + + depends_on = [ + azurerm_network_interface.nic, + azurerm_network_security_group.nsg + ] +} + + +resource "azurerm_virtual_machine_extension" "azure_monitor_agent" { + name = "${var.name}MonitorAgent" + virtual_machine_id = azurerm_linux_virtual_machine.virtual_machine.id + publisher = "Microsoft.Azure.Monitor" + type = "AzureMonitorLinuxAgent" + type_handler_version = "1.21" + auto_upgrade_minor_version = true + automatic_upgrade_enabled = true + tags = var.tags + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_monitor_data_collection_rule" "linux" { + name = "LinuxVmMonitorDataCollectionRule" + resource_group_name = var.resource_group_name + location = var.location + tags = var.tags + + destinations { + log_analytics { + workspace_resource_id = var.log_analytics_workspace_resource_id + name = "default" + } + } + + data_flow { + streams = ["Microsoft-InsightsMetrics", "Microsoft-Syslog", "Microsoft-Perf"] + destinations = ["default"] + } + + data_sources { + syslog { + facility_names = ["*"] + log_levels = ["*"] + name = "syslog" + } + + performance_counter { + streams = ["Microsoft-Perf", "Microsoft-InsightsMetrics"] + sampling_frequency_in_seconds = 60 + name = "perfcounter" + counter_specifiers = [ + "\\Processor Information(_Total)\\% Processor Time", + "\\Processor Information(_Total)\\% Privileged Time", + "\\Processor Information(_Total)\\% User Time", + "\\Processor Information(_Total)\\Processor Frequency", + "\\System\\Processes", + "\\Process(_Total)\\Thread Count", + "\\Process(_Total)\\Handle Count", + "\\System\\System Up Time", + "\\System\\Context Switches/sec", + "\\System\\Processor Queue Length", + "\\Memory\\% Committed Bytes In Use", + "\\Memory\\Available Bytes", + "\\Memory\\Committed Bytes", + "\\Memory\\Cache Bytes", + "\\Memory\\Pool Paged Bytes", + "\\Memory\\Pool Nonpaged Bytes", + "\\Memory\\Pages/sec", + "\\Memory\\Page Faults/sec", + "\\Process(_Total)\\Working Set", + "\\Process(_Total)\\Working Set - Private", + "\\LogicalDisk(_Total)\\% Disk Time", + "\\LogicalDisk(_Total)\\% Disk Read Time", + "\\LogicalDisk(_Total)\\% Disk Write Time", + "\\LogicalDisk(_Total)\\% Idle Time", + "\\LogicalDisk(_Total)\\Disk Bytes/sec", + "\\LogicalDisk(_Total)\\Disk Read Bytes/sec", + "\\LogicalDisk(_Total)\\Disk Write Bytes/sec", + "\\LogicalDisk(_Total)\\Disk Transfers/sec", + "\\LogicalDisk(_Total)\\Disk Reads/sec", + "\\LogicalDisk(_Total)\\Disk Writes/sec", + "\\LogicalDisk(_Total)\\Avg. Disk sec/Transfer", + "\\LogicalDisk(_Total)\\Avg. Disk sec/Read", + "\\LogicalDisk(_Total)\\Avg. Disk sec/Write", + "\\LogicalDisk(_Total)\\Avg. Disk Queue Length", + "\\LogicalDisk(_Total)\\Avg. Disk Read Queue Length", + "\\LogicalDisk(_Total)\\Avg. Disk Write Queue Length", + "\\LogicalDisk(_Total)\\% Free Space", + "\\LogicalDisk(_Total)\\Free Megabytes", + "\\Network Interface(*)\\Bytes Total/sec", + "\\Network Interface(*)\\Bytes Sent/sec", + "\\Network Interface(*)\\Bytes Received/sec", + "\\Network Interface(*)\\Packets/sec", + "\\Network Interface(*)\\Packets Sent/sec", + "\\Network Interface(*)\\Packets Received/sec", + "\\Network Interface(*)\\Packets Outbound Errors", + "\\Network Interface(*)\\Packets Received Errors", + ] + } + } + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_monitor_data_collection_rule_association" "virtual_machine_association" { + name = "LinuxVmMonitorDataCollectionRuleAssociation" + target_resource_id = azurerm_linux_virtual_machine.virtual_machine.id + data_collection_rule_id = azurerm_monitor_data_collection_rule.linux.id +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/outputs.tf new file mode 100644 index 000000000..a7ac8a18b --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/outputs.tf @@ -0,0 +1,9 @@ +output "public_ip" { + description = "Specifies the public IP address of the virtual machine" + value = azurerm_linux_virtual_machine.virtual_machine.public_ip_address +} + +output "username" { + description = "Specifies the username of the virtual machine" + value = var.vm_user +} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/variables.tf new file mode 100644 index 000000000..b3a2adbc9 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/variables.tf @@ -0,0 +1,95 @@ +variable resource_group_name { + description = "(Required) Specifies the resource group name of the virtual machine" + type = string +} + +variable name { + description = "(Required) Specifies the name of the virtual machine" + type = string +} + +variable size { + description = "(Required) Specifies the size of the virtual machine" + type = string +} + +variable "os_disk_image" { + type = map(string) + description = "(Optional) Specifies the os disk image of the virtual machine" + default = { + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts-gen2" + version = "latest" + } +} + +variable "os_disk_storage_account_type" { + description = "(Optional) Specifies the storage account type of the os disk of the virtual machine" + default = "StandardSSD_LRS" + type = string + + validation { + condition = contains(["Premium_LRS", "Premium_ZRS", "StandardSSD_LRS", "StandardSSD_ZRS", "Standard_LRS"], var.os_disk_storage_account_type) + error_message = "The storage account type of the OS disk is invalid." + } +} + +variable public_ip { + description = "(Optional) Specifies whether create a public IP for the virtual machine" + type = bool + default = false +} + +variable location { + description = "(Required) Specifies the location of the virtual machine" + type = string +} + +variable subnet_id { + description = "(Required) Specifies the resource id of the subnet hosting the virtual machine" + type = string +} + +variable vm_user { + description = "(Required) Specifies the username of the virtual machine" + type = string + default = "azadmin" +} + +variable "boot_diagnostics_storage_account" { + description = "(Optional) The Primary/Secondary Endpoint for the Azure Storage Account (general purpose) which should be used to store Boot Diagnostics, including Console Output and Screenshots from the Hypervisor." + default = null +} + +variable "tags" { + description = "(Optional) Specifies the tags of the storage account" + default = {} +} + +variable "log_analytics_workspace_id" { + description = "Specifies the log analytics workspace id" + type = string +} + +variable "log_analytics_workspace_key" { + description = "Specifies the log analytics workspace key" + type = string +} + +variable "log_analytics_workspace_resource_id" { + description = "Specifies the log analytics workspace resource id" + type = string +} + + +variable "log_analytics_retention_days" { + description = "Specifies the number of days of the retention policy" + type = number + default = 7 +} + +variable "admin_ssh_public_key" { + description = "Specifies the public SSH key" + type = string +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf new file mode 100644 index 000000000..cf3cd7bcd --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf @@ -0,0 +1,59 @@ +resource "azurerm_virtual_network" "vnet" { + name = var.vnet_name + address_space = var.address_space + location = var.location + resource_group_name = var.resource_group_name + tags = var.tags + + lifecycle { + ignore_changes = [ + tags + ] + } +} + +resource "azurerm_subnet" "subnet" { + for_each = { for subnet in var.subnets : subnet.name => subnet } + + name = each.key + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = each.value.address_prefixes + private_endpoint_network_policies_enabled = each.value.private_endpoint_network_policies_enabled + private_link_service_network_policies_enabled = each.value.private_link_service_network_policies_enabled + + dynamic "delegation" { + for_each = each.value.delegation != null ? [each.value.delegation] : [] + content { + name = "delegation" + + service_delegation { + name = delegation.value + } + } + } +} + +resource "azurerm_monitor_diagnostic_setting" "settings" { + name = "DiagnosticsSettings" + target_resource_id = azurerm_virtual_network.vnet.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "VMProtectionAlerts" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } + + metric { + category = "AllMetrics" + + retention_policy { + enabled = true + days = var.log_analytics_retention_days + } + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf new file mode 100644 index 000000000..4f0e02711 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf @@ -0,0 +1,19 @@ +output "name" { + description = "Specifies the name of the virtual network" + value = azurerm_virtual_network.vnet.name +} + +output "vnet_id" { + description = "Specifies the resource id of the virtual network" + value = azurerm_virtual_network.vnet.id +} + +output "subnet_ids" { + description = "Contains a list of the the resource id of the subnets" + value = { for subnet in azurerm_subnet.subnet : subnet.name => subnet.id } +} + +output "subnet_ids_as_list" { + description = "Returns the list of the subnet ids as a list of strings." + value = [ for subnet in azurerm_subnet.subnet : subnet.id ] +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf new file mode 100644 index 000000000..6252c0c1b --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf @@ -0,0 +1,46 @@ +variable "resource_group_name" { + description = "Resource Group name" + type = string +} + +variable "location" { + description = "Location in which to deploy the network" + type = string +} + +variable "vnet_name" { + description = "VNET name" + type = string +} + +variable "address_space" { + description = "VNET address space" + type = list(string) +} + +variable "subnets" { + description = "Subnets configuration" + type = list(object({ + name = string + address_prefixes = list(string) + private_endpoint_network_policies_enabled = bool + private_link_service_network_policies_enabled = bool + delegation = string + })) +} + +variable "tags" { + description = "(Optional) Specifies the tags of the storage account" + default = {} +} + +variable "log_analytics_workspace_id" { + description = "Specifies the log analytics workspace id" + type = string +} + +variable "log_analytics_retention_days" { + description = "Specifies the number of days of the retention policy" + type = number + default = 7 +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/main.tf new file mode 100644 index 000000000..ea60dd098 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/main.tf @@ -0,0 +1,17 @@ +resource "azurerm_virtual_network_peering" "peering" { + name = var.peering_name_1_to_2 + resource_group_name = var.vnet_1_rg + virtual_network_name = var.vnet_1_name + remote_virtual_network_id = var.vnet_2_id + allow_virtual_network_access = true + allow_forwarded_traffic = true +} + +resource "azurerm_virtual_network_peering" "peering-back" { + name = var.peering_name_2_to_1 + resource_group_name = var.vnet_2_rg + virtual_network_name = var.vnet_2_name + remote_virtual_network_id = var.vnet_1_id + allow_virtual_network_access = true + allow_forwarded_traffic = true +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/variables.tf new file mode 100644 index 000000000..9bb640f25 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/variables.tf @@ -0,0 +1,41 @@ +variable "vnet_1_name" { + description = "Specifies the name of the first virtual network" + type = string +} + +variable "vnet_1_id" { + description = "Specifies the resource id of the first virtual network" + type = string +} + +variable "vnet_1_rg" { + description = "Specifies the resource group name of the first virtual network" + type = string +} + +variable "vnet_2_name" { + description = "Specifies the name of the second virtual network" + type = string +} + +variable "vnet_2_id" { + description = "Specifies the resource id of the second virtual network" + type = string +} + +variable "vnet_2_rg" { + description = "Specifies the resource group name of the second virtual network" + type = string +} + +variable "peering_name_1_to_2" { + description = "(Optional) Specifies the name of the first to second virtual network peering" + type = string + default = "peering1to2" +} + +variable "peering_name_2_to_1" { + description = "(Optional) Specifies the name of the second to first virtual network peering" + type = string + default = "peering2to1" +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf new file mode 100644 index 000000000..e69de29bb diff --git a/scenarios/AksOpenAiTerraform/terraform/register-preview-features.sh b/scenarios/AksOpenAiTerraform/terraform/register-preview-features.sh new file mode 100644 index 000000000..af015f216 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/register-preview-features.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Install aks-preview Azure extension +echo "Checking if [aks-preview] extension is already installed..." +az extension show --name aks-preview &>/dev/null + +if [[ $? == 0 ]]; then + echo "[aks-preview] extension is already installed" + + # Update the extension to make sure you have the latest version installed + echo "Updating [aks-preview] extension..." + az extension update --name aks-preview &>/dev/null +else + echo "[aks-preview] extension is not installed. Installing..." + + # Install aks-preview extension + az extension add --name aks-preview 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[aks-preview] extension successfully installed" + else + echo "Failed to install [aks-preview] extension" + exit + fi +fi + +# Registering AKS features +aksExtensions=( + "AzureServiceMeshPreview" + "AKS-KedaPreview" + "RunCommandPreview" + "EnableOIDCIssuerPreview" + "EnableWorkloadIdentityPreview" + "EnableImageCleanerPreview" +"AKS-VPAPreview") +ok=0 +registeringExtensions=() +for aksExtension in ${aksExtensions[@]}; do + echo "Checking if [$aksExtension] extension is already registered..." + extension=$(az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService/$aksExtension') && @.properties.state == 'Registered'].{Name:name}" --output tsv) + if [[ -z $extension ]]; then + echo "[$aksExtension] extension is not registered." + echo "Registering [$aksExtension] extension..." + az feature register --name $aksExtension --namespace Microsoft.ContainerService + registeringExtensions+=("$aksExtension") + ok=1 + else + echo "[$aksExtension] extension is already registered." + fi +done +echo $registeringExtensions +delay=1 +for aksExtension in ${registeringExtensions[@]}; do + echo -n "Checking if [$aksExtension] extension is already registered..." + while true; do + extension=$(az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService/$aksExtension') && @.properties.state == 'Registered'].{Name:name}" --output tsv) + if [[ -z $extension ]]; then + echo -n "." + sleep $delay + else + echo "." + break + fi + done +done + +if [[ $ok == 1 ]]; then + echo "Refreshing the registration of the Microsoft.ContainerService resource provider..." + az provider register --namespace Microsoft.ContainerService + echo "Microsoft.ContainerService resource provider registration successfully refreshed" +fi \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/terraform.tfvars b/scenarios/AksOpenAiTerraform/terraform/terraform.tfvars new file mode 100644 index 000000000..db5be9172 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/terraform.tfvars @@ -0,0 +1,9 @@ +name_prefix = "magic8ball" +domain = "contoso.com" +subdomain = "magic" +namespace = "magic8ball" +service_account_name = "magic8ball-sa" +ssh_public_key = "XXXXXXX" +vm_enabled = true +location = "westeurope" +admin_group_object_ids = ["XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"] \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf new file mode 100644 index 000000000..31d28bf69 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -0,0 +1,743 @@ +variable "name_prefix" { + description = "(Optional) A prefix for the name of all the resource groups and resources." + type = string + default = "Bingo" + nullable = true +} + +variable "log_analytics_workspace_name" { + description = "Specifies the name of the log analytics workspace" + default = "Workspace" + type = string +} + +variable "log_analytics_retention_days" { + description = "Specifies the number of days of the retention policy" + type = number + default = 30 +} + +variable "solution_plan_map" { + description = "Specifies solutions to deploy to log analytics workspace" + default = { + ContainerInsights= { + product = "OMSGallery/ContainerInsights" + publisher = "Microsoft" + } + } + type = map(any) +} + +variable "location" { + description = "Specifies the location for the resource group and all the resources" + default = "northeurope" + type = string +} + +variable "resource_group_name" { + description = "Specifies the resource group name" + default = "RG" + type = string +} + +variable "vnet_name" { + description = "Specifies the name of the AKS subnet" + default = "AksVNet" + type = string +} + +variable "vnet_address_space" { + description = "Specifies the address prefix of the AKS subnet" + default = ["10.0.0.0/8"] + type = list(string) +} + +variable "system_node_pool_subnet_name" { + description = "Specifies the name of the subnet that hosts the system node pool" + default = "SystemSubnet" + type = string +} + +variable "system_node_pool_subnet_address_prefix" { + description = "Specifies the address prefix of the subnet that hosts the system node pool" + default = ["10.240.0.0/16"] + type = list(string) +} + +variable "user_node_pool_subnet_name" { + description = "Specifies the name of the subnet that hosts the user node pool" + default = "UserSubnet" + type = string +} + +variable "user_node_pool_subnet_address_prefix" { + description = "Specifies the address prefix of the subnet that hosts the user node pool" + type = list(string) + default = ["10.241.0.0/16"] +} + +variable "pod_subnet_name" { + description = "Specifies the name of the jumpbox subnet" + default = "PodSubnet" + type = string +} + +variable "pod_subnet_address_prefix" { + description = "Specifies the address prefix of the jumbox subnet" + default = ["10.242.0.0/16"] + type = list(string) +} + +variable "vm_subnet_name" { + description = "Specifies the name of the jumpbox subnet" + default = "VmSubnet" + type = string +} + +variable "vm_subnet_address_prefix" { + description = "Specifies the address prefix of the jumbox subnet" + default = ["10.243.1.0/24"] + type = list(string) +} + +variable "bastion_subnet_address_prefix" { + description = "Specifies the address prefix of the firewall subnet" + default = ["10.243.2.0/24"] + type = list(string) +} + +variable "aks_cluster_name" { + description = "(Required) Specifies the name of the AKS cluster." + default = "Aks" + type = string +} + +variable "private_cluster_enabled" { + description = "(Optional) Specifies wether the AKS cluster be private or not." + default = false + type = bool +} + +variable "role_based_access_control_enabled" { + description = "(Required) Is Role Based Access Control Enabled? Changing this forces a new resource to be created." + default = true + type = bool +} + +variable "automatic_channel_upgrade" { + description = "(Optional) The upgrade channel for this Kubernetes Cluster. Possible values are patch, rapid, and stable." + default = "stable" + type = string + + validation { + condition = contains( ["patch", "rapid", "stable"], var.automatic_channel_upgrade) + error_message = "The upgrade mode is invalid." + } +} + +variable "admin_group_object_ids" { + description = "(Optional) A list of Object IDs of Azure Active Directory Groups which should have Admin Role on the Cluster." + default = [] + type = list(string) +} + +variable "azure_rbac_enabled" { + description = "(Optional) Is Role Based Access Control based on Azure AD enabled?" + default = true + type = bool +} + +variable "sku_tier" { + description = "(Optional) The SKU Tier that should be used for this Kubernetes Cluster. Possible values are Free and Paid (which includes the Uptime SLA). Defaults to Free." + default = "Free" + type = string + + validation { + condition = contains( ["Free", "Paid"], var.sku_tier) + error_message = "The sku tier is invalid." + } +} + +variable "kubernetes_version" { + description = "Specifies the AKS Kubernetes version" + default = "1.26.3" + type = string +} + +variable "system_node_pool_vm_size" { + description = "Specifies the vm size of the system node pool" + default = "Standard_F8s_v2" + type = string +} + +variable "system_node_pool_availability_zones" { + description = "Specifies the availability zones of the system node pool" + default = ["1", "2", "3"] + type = list(string) +} + +variable "network_dns_service_ip" { + description = "Specifies the DNS service IP" + default = "10.2.0.10" + type = string +} + +variable "network_service_cidr" { + description = "Specifies the service CIDR" + default = "10.2.0.0/24" + type = string +} + +variable "network_plugin" { + description = "Specifies the network plugin of the AKS cluster" + default = "azure" + type = string +} + +variable "system_node_pool_name" { + description = "Specifies the name of the system node pool" + default = "system" + type = string +} + +variable "system_node_pool_enable_auto_scaling" { + description = "(Optional) Whether to enable auto-scaler. Defaults to false." + type = bool + default = true +} + +variable "system_node_pool_enable_host_encryption" { + description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." + type = bool + default = false +} + +variable "system_node_pool_enable_node_public_ip" { + description = "(Optional) Should each node have a Public IP Address? Defaults to false. Changing this forces a new resource to be created." + type = bool + default = false +} + +variable "system_node_pool_max_pods" { + description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." + type = number + default = 50 +} + +variable "system_node_pool_node_labels" { + description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." + type = map(any) + default = {} +} + +variable "system_node_pool_node_taints" { + description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." + type = list(string) + default = ["CriticalAddonsOnly=true:NoSchedule"] +} + +variable "system_node_pool_os_disk_type" { + description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." + type = string + default = "Ephemeral" +} + +variable "system_node_pool_max_count" { + description = "(Required) The maximum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be greater than or equal to min_count." + type = number + default = 10 +} + +variable "system_node_pool_min_count" { + description = "(Required) The minimum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be less than or equal to max_count." + type = number + default = 3 +} + +variable "system_node_pool_node_count" { + description = "(Optional) The initial number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be a value in the range min_count - max_count." + type = number + default = 3 +} + +variable "user_node_pool_name" { + description = "(Required) Specifies the name of the node pool." + type = string + default = "user" +} + +variable "user_node_pool_vm_size" { + description = "(Required) The SKU which should be used for the Virtual Machines used in this Node Pool. Changing this forces a new resource to be created." + type = string + default = "Standard_F8s_v2" +} + +variable "user_node_pool_availability_zones" { + description = "(Optional) A list of Availability Zones where the Nodes in this Node Pool should be created in. Changing this forces a new resource to be created." + type = list(string) + default = ["1", "2", "3"] +} + +variable "user_node_pool_enable_auto_scaling" { + description = "(Optional) Whether to enable auto-scaler. Defaults to false." + type = bool + default = true +} + +variable "user_node_pool_enable_host_encryption" { + description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." + type = bool + default = false +} + +variable "user_node_pool_enable_node_public_ip" { + description = "(Optional) Should each node have a Public IP Address? Defaults to false. Changing this forces a new resource to be created." + type = bool + default = false +} + +variable "user_node_pool_max_pods" { + description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." + type = number + default = 50 +} + +variable "user_node_pool_mode" { + description = "(Optional) Should this Node Pool be used for System or User resources? Possible values are System and User. Defaults to User." + type = string + default = "User" +} + +variable "user_node_pool_node_labels" { + description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." + type = map(any) + default = {} +} + +variable "user_node_pool_node_taints" { + description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." + type = list(string) + default = [] +} + +variable "user_node_pool_os_disk_type" { + description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." + type = string + default = "Ephemeral" +} + +variable "user_node_pool_os_type" { + description = "(Optional) The Operating System which should be used for this Node Pool. Changing this forces a new resource to be created. Possible values are Linux and Windows. Defaults to Linux." + type = string + default = "Linux" +} + +variable "user_node_pool_priority" { + description = "(Optional) The Priority for Virtual Machines within the Virtual Machine Scale Set that powers this Node Pool. Possible values are Regular and Spot. Defaults to Regular. Changing this forces a new resource to be created." + type = string + default = "Regular" +} + +variable "user_node_pool_max_count" { + description = "(Required) The maximum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be greater than or equal to min_count." + type = number + default = 10 +} + +variable "user_node_pool_min_count" { + description = "(Required) The minimum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be less than or equal to max_count." + type = number + default = 3 +} + +variable "user_node_pool_node_count" { + description = "(Optional) The initial number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be a value in the range min_count - max_count." + type = number + default = 3 +} + +variable "vm_enabled" { + description = "(Optional) Specifies whether create a virtual machine" + type = bool + default = false +} + +variable "vm_name" { + description = "Specifies the name of the jumpbox virtual machine" + default = "Vm" + type = string +} + +variable "vm_public_ip" { + description = "(Optional) Specifies whether create a public IP for the virtual machine" + type = bool + default = false +} + +variable "vm_size" { + description = "Specifies the size of the jumpbox virtual machine" + default = "Standard_DS1_v2" + type = string +} + +variable "vm_os_disk_storage_account_type" { + description = "Specifies the storage account type of the os disk of the jumpbox virtual machine" + default = "Premium_LRS" + type = string + + validation { + condition = contains(["Premium_LRS", "Premium_ZRS", "StandardSSD_LRS", "StandardSSD_ZRS", "Standard_LRS"], var.vm_os_disk_storage_account_type) + error_message = "The storage account type of the OS disk is invalid." + } +} + +variable "vm_os_disk_image" { + type = map(string) + description = "Specifies the os disk image of the virtual machine" + default = { + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts-gen2" + version = "latest" + } +} + +variable "storage_account_kind" { + description = "(Optional) Specifies the account kind of the storage account" + default = "StorageV2" + type = string + + validation { + condition = contains(["Storage", "StorageV2"], var.storage_account_kind) + error_message = "The account kind of the storage account is invalid." + } +} + +variable "storage_account_tier" { + description = "(Optional) Specifies the account tier of the storage account" + default = "Standard" + type = string + + validation { + condition = contains(["Standard", "Premium"], var.storage_account_tier) + error_message = "The account tier of the storage account is invalid." + } +} + +variable "acr_name" { + description = "Specifies the name of the container registry" + type = string + default = "Acr" +} + +variable "acr_sku" { + description = "Specifies the name of the container registry" + type = string + default = "Premium" + + validation { + condition = contains(["Basic", "Standard", "Premium"], var.acr_sku) + error_message = "The container registry sku is invalid." + } +} + +variable "acr_admin_enabled" { + description = "Specifies whether admin is enabled for the container registry" + type = bool + default = true +} + +variable "acr_georeplication_locations" { + description = "(Optional) A list of Azure locations where the container registry should be geo-replicated." + type = list(string) + default = [] +} + +variable "tags" { + description = "(Optional) Specifies tags for all the resources" + default = { + createdWith = "Terraform" + } +} + +variable "bastion_host_name" { + description = "(Optional) Specifies the name of the bastion host" + default = "BastionHost" + type = string +} + +variable "storage_account_replication_type" { + description = "(Optional) Specifies the replication type of the storage account" + default = "LRS" + type = string + + validation { + condition = contains(["LRS", "ZRS", "GRS", "GZRS", "RA-GRS", "RA-GZRS"], var.storage_account_replication_type) + error_message = "The replication type of the storage account is invalid." + } +} + +variable "key_vault_name" { + description = "Specifies the name of the key vault." + type = string + default = "KeyVault" +} + +variable "key_vault_sku_name" { + description = "(Required) The Name of the SKU used for this Key Vault. Possible values are standard and premium." + type = string + default = "standard" + + validation { + condition = contains(["standard", "premium" ], var.key_vault_sku_name) + error_message = "The sku name of the key vault is invalid." + } +} + +variable"key_vault_enabled_for_deployment" { + description = "(Optional) Boolean flag to specify whether Azure Virtual Machines are permitted to retrieve certificates stored as secrets from the key vault. Defaults to false." + type = bool + default = true +} + +variable"key_vault_enabled_for_disk_encryption" { + description = " (Optional) Boolean flag to specify whether Azure Disk Encryption is permitted to retrieve secrets from the vault and unwrap keys. Defaults to false." + type = bool + default = true +} + +variable"key_vault_enabled_for_template_deployment" { + description = "(Optional) Boolean flag to specify whether Azure Resource Manager is permitted to retrieve secrets from the key vault. Defaults to false." + type = bool + default = true +} + +variable"key_vault_enable_rbac_authorization" { + description = "(Optional) Boolean flag to specify whether Azure Key Vault uses Role Based Access Control (RBAC) for authorization of data actions. Defaults to false." + type = bool + default = true +} + +variable"key_vault_purge_protection_enabled" { + description = "(Optional) Is Purge Protection enabled for this Key Vault? Defaults to false." + type = bool + default = false +} + +variable "key_vault_soft_delete_retention_days" { + description = "(Optional) The number of days that items should be retained for once soft-deleted. This value can be between 7 and 90 (the default) days." + type = number + default = 30 +} + +variable "key_vault_bypass" { + description = "(Required) Specifies which traffic can bypass the network rules. Possible values are AzureServices and None." + type = string + default = "AzureServices" + + validation { + condition = contains(["AzureServices", "None" ], var.key_vault_bypass) + error_message = "The valut of the bypass property of the key vault is invalid." + } +} + +variable "key_vault_default_action" { + description = "(Required) The Default Action to use when no rules match from ip_rules / virtual_network_subnet_ids. Possible values are Allow and Deny." + type = string + default = "Allow" + + validation { + condition = contains(["Allow", "Deny" ], var.key_vault_default_action) + error_message = "The value of the default action property of the key vault is invalid." + } +} + +variable "admin_username" { + description = "(Required) Specifies the admin username of the jumpbox virtual machine and AKS worker nodes." + type = string + default = "azadmin" +} + +variable "ssh_public_key" { + description = "(Required) Specifies the SSH public key for the jumpbox virtual machine and AKS worker nodes." + type = string +} + +variable "keda_enabled" { + description = "(Optional) Specifies whether KEDA Autoscaler can be used for workloads." + type = bool + default = true +} + +variable "vertical_pod_autoscaler_enabled" { + description = "(Optional) Specifies whether Vertical Pod Autoscaler should be enabled." + type = bool + default = true +} + +variable "workload_identity_enabled" { + description = "(Optional) Specifies whether Azure AD Workload Identity should be enabled for the Cluster. Defaults to false." + type = bool + default = true +} + +variable "oidc_issuer_enabled" { + description = "(Optional) Enable or Disable the OIDC issuer URL." + type = bool + default = true +} + +variable "open_service_mesh_enabled" { + description = "(Optional) Is Open Service Mesh enabled? For more details, please visit Open Service Mesh for AKS." + type = bool + default = true +} + +variable "image_cleaner_enabled" { + description = "(Optional) Specifies whether Image Cleaner is enabled." + type = bool + default = true +} + +variable "azure_policy_enabled" { + description = "(Optional) Should the Azure Policy Add-On be enabled? For more details please visit Understand Azure Policy for Azure Kubernetes Service" + type = bool + default = true +} + +variable "http_application_routing_enabled" { + description = "(Optional) Should HTTP Application Routing be enabled?" + type = bool + default = false +} + +variable "openai_name" { + description = "(Required) Specifies the name of the Azure OpenAI Service" + type = string + default = "OpenAi" +} + +variable "openai_sku_name" { + description = "(Optional) Specifies the sku name for the Azure OpenAI Service" + type = string + default = "S0" +} + +variable "openai_custom_subdomain_name" { + description = "(Optional) Specifies the custom subdomain name of the Azure OpenAI Service" + type = string + nullable = true + default = "" +} + +variable "openai_public_network_access_enabled" { + description = "(Optional) Specifies whether public network access is allowed for the Azure OpenAI Service" + type = bool + default = true +} + +variable "openai_deployments" { + description = "(Optional) Specifies the deployments of the Azure OpenAI Service" + type = list(object({ + name = string + model = object({ + name = string + version = string + }) + rai_policy_name = string + })) + default = [ + { + name = "gpt-35-turbo" + model = { + name = "gpt-35-turbo" + version = "0301" + } + rai_policy_name = "" + } + ] +} + +variable "nat_gateway_name" { + description = "(Required) Specifies the name of the Azure OpenAI Service" + type = string + default = "NatGateway" +} + +variable "nat_gateway_sku_name" { + description = "(Optional) The SKU which should be used. At this time the only supported value is Standard. Defaults to Standard" + type = string + default = "Standard" +} + +variable "nat_gateway_idle_timeout_in_minutes" { + description = "(Optional) The idle timeout which should be used in minutes. Defaults to 4." + type = number + default = 4 +} + +variable "nat_gateway_zones" { + description = " (Optional) A list of Availability Zones in which this NAT Gateway should be located. Changing this forces a new NAT Gateway to be created." + type = list(string) + default = ["1"] +} + +variable "workload_managed_identity_name" { + description = "(Required) Specifies the name of the workload user-defined managed identity." + type = string + default = "WorkloadManagedIdentity" +} + +variable "subdomain" { + description = "Specifies the subdomain of the Kubernetes ingress object." + type = string + default = "magic8ball" +} + +variable "domain" { + description = "Specifies the domain of the Kubernetes ingress object." + type = string + default = "contoso.com" +} + +variable "namespace" { + description = "Specifies the namespace of the workload application that accesses the Azure OpenAI Service." + type = string + default = "magic8ball" +} + +variable "service_account_name" { + description = "Specifies the name of the service account of the workload application that accesses the Azure OpenAI Service." + type = string + default = "magic8ball-sa" +} + +variable "email" { + description = "Specifies the email address for the cert-manager cluster issuer." + type = string + default = "paolos@microsoft.com" +} + +variable "deployment_script_name" { + description = "(Required) Specifies the name of the Azure OpenAI Service" + type = string + default = "BashScript" +} + +variable "deployment_script_azure_cli_version" { + description = "(Required) Azure CLI module version to be used." + type = string + default = "2.9.1" +} + +variable "deployment_script_managed_identity_name" { + description = "Specifies the name of the user-defined managed identity used by the deployment script." + type = string + default = "ScriptManagedIdentity" +} + +variable "deployment_script_primary_script_uri" { + description = "(Optional) Uri for the script. This is the entry point for the external script. Changing this forces a new Resource Deployment Script to be created." + type = string + default = "https://paolosalvatori.blob.core.windows.net/scripts/install-nginx-via-helm-and-create-sa.sh" +} \ No newline at end of file From 1fc394337480503befd0c3746d45da274d5bb99f Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 26 Nov 2024 05:39:43 -0500 Subject: [PATCH 002/119] Init terraform --- .../terraform/.terraform.lock.hcl | 43 +++++++++++++++++++ .../terraform/terraform.tfvars | 9 ---- 2 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl delete mode 100644 scenarios/AksOpenAiTerraform/terraform/terraform.tfvars diff --git a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl new file mode 100644 index 000000000..d3369d32b --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl @@ -0,0 +1,43 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.58.0" + constraints = "3.58.0" + hashes = [ + "h1:Hvlt3hgTiip6xMeq8/EDGqF8NoVuZjYdTZdO79YNXsw=", + "h1:ceZlVBDs02TjOxY4JGLaeqCigsy7gcEPLcJudiTurb4=", + "zh:22b19802605ca3e2b811e33650438be3647748cf8f75474c78448c30ac1cad0b", + "zh:402ce010f4b68337abaccf8059c37294cabcbdbc3cefd9491dcd312e36ceea3c", + "zh:53d2cd15f1631c7ffb47918064d644899cc671d47c72f4dafee4e2a5e69afd14", + "zh:5a6b1c55629cff555472d1d43ad6e802693f7fd046c7d37718d4de6f52dbf66b", + "zh:6181dccb7bca7cd84b0295a0332f19a7347a9586101f0a5e51b53bda1ec74651", + "zh:854181d6a8821b3707775c913e91dd7944fcb55098953ef030168fa3cd0224aa", + "zh:b44c758424d1a037fd833e0c69b29e3ac4047ab95653bb3e080835e55bd9badb", + "zh:b6ee916a1579bba29b1aacce8897c6733fa97ba0dba2808f1ffa9ab492743fab", + "zh:b7ab57044649578410dadfdf4412fc5f8aa085a25ea0b061393e843b49b43b63", + "zh:cb68ddb922eb4be74dedf58c953d7f778b4e5f3cdcbe2ea83e02b12296ce4969", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fe9e86173134cd9dc8ed65eae8634abc6d6f6806b5b412f54fccf4047052daa0", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.3" + hashes = [ + "h1:Fnaec9vA8sZ8BXVlN3Xn9Jz3zghSETIKg7ch8oXhxno=", + "h1:N2IQabOiZC5eCEGrfgVS6ChVmRDh1ENtfHgGjnV4QQQ=", + "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", + "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8", + "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe", + "zh:4fa45c44c0de582c2edb8a2e054f55124520c16a39b2dfc0355929063b6395b1", + "zh:588508280501a06259e023b0695f6a18149a3816d259655c424d068982cbdd36", + "zh:737c4d99a87d2a4d1ac0a54a73d2cb62974ccb2edbd234f333abd079a32ebc9e", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a357ab512e5ebc6d1fda1382503109766e21bbfdfaa9ccda43d313c122069b30", + "zh:c51bfb15e7d52cc1a2eaec2a903ac2aff15d162c172b1b4c17675190e8147615", + "zh:e0951ee6fa9df90433728b96381fb867e3db98f66f735e0c3e24f8f16903f0ad", + "zh:e3cdcb4e73740621dabd82ee6a37d6cfce7fee2a03d8074df65086760f5cf556", + "zh:eff58323099f1bd9a0bec7cb04f717e7f1b2774c7d612bf7581797e1622613a0", + ] +} diff --git a/scenarios/AksOpenAiTerraform/terraform/terraform.tfvars b/scenarios/AksOpenAiTerraform/terraform/terraform.tfvars deleted file mode 100644 index db5be9172..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/terraform.tfvars +++ /dev/null @@ -1,9 +0,0 @@ -name_prefix = "magic8ball" -domain = "contoso.com" -subdomain = "magic" -namespace = "magic8ball" -service_account_name = "magic8ball-sa" -ssh_public_key = "XXXXXXX" -vm_enabled = true -location = "westeurope" -admin_group_object_ids = ["XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"] \ No newline at end of file From 4087f1d0546cfc1c88cd8ddd2567e67494c53458 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 26 Nov 2024 05:39:54 -0500 Subject: [PATCH 003/119] Change readme --- scenarios/AksOpenAiTerraform/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index 360ebc9b7..0cb1ae9a4 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -13,6 +13,7 @@ ms.custom: innovation-engine, linux-related-content Run commands below to set up AKS extensions for Azure. ```bash -./terraform/register-preview-features.sh +# ./terraform/register-preview-features.sh +echo "HI" ``` From af736b299d69732e0e78889292c8384455296d4c Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 26 Nov 2024 05:45:33 -0500 Subject: [PATCH 004/119] Add gitignore --- scenarios/AksOpenAiTerraform/.gitignore | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 scenarios/AksOpenAiTerraform/.gitignore diff --git a/scenarios/AksOpenAiTerraform/.gitignore b/scenarios/AksOpenAiTerraform/.gitignore new file mode 100644 index 000000000..21e6d3cbd --- /dev/null +++ b/scenarios/AksOpenAiTerraform/.gitignore @@ -0,0 +1,37 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Ignore transient lock info files created by terraform apply +.terraform.tfstate.lock.info + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc \ No newline at end of file From 63a2f0c9ec2ff56a3388a18eef279d095876ec4c Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 26 Nov 2024 05:59:18 -0500 Subject: [PATCH 005/119] Add metadata --- scenarios/AksOpenAiTerraform/README.md | 3 +-- scenarios/metadata.json | 10 ++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index 0cb1ae9a4..b046df346 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -15,5 +15,4 @@ Run commands below to set up AKS extensions for Azure. ```bash # ./terraform/register-preview-features.sh echo "HI" -``` - +``` \ No newline at end of file diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 5dfa2d3df..7cd8152d6 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -272,5 +272,15 @@ "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/azure-aks-docs/refs/heads/main/articles/aks/ai-toolchain-operator.md", "documentationUrl": "", "configurations": {} + }, + { + "status": "active", + "key": "AksOpenAiTerraform/README.md", + "title": "How to deploy and run an Azure OpenAI ChatGPT application on AKS via Terraform", + "description": "This article shows how to deploy an AKS cluster and Azure OpenAI Service via Terraform and how to deploy a ChatGPT-like application in Python.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/refs/heads/test_terraform/scenarios/AksOpenAiTerraform/README.md", + "documentationUrl": "", + "configurations": {} } ] From 036fa6fe7a1aa82d15b49bf7b40c13249f37789c Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 26 Nov 2024 15:51:01 -0500 Subject: [PATCH 006/119] Remove jumpbox VM --- .../AksOpenAiTerraform/terraform/main.tf | 21 -- .../terraform/modules/virtual_machine/main.tf | 221 ------------------ .../modules/virtual_machine/outputs.tf | 9 - .../modules/virtual_machine/variables.tf | 95 -------- .../AksOpenAiTerraform/terraform/variables.tf | 51 ---- 5 files changed, 397 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/variables.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index e8ed5536b..421ed68e7 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -281,27 +281,6 @@ module "bastion_host" { tags = var.tags } -module "virtual_machine" { - count = var.vm_enabled ? 1 : 0 - source = "./modules/virtual_machine" - name = var.name_prefix == null ? "${random_string.prefix.result}${var.vm_name}" : "${var.name_prefix}${var.vm_name}" - size = var.vm_size - location = var.location - public_ip = var.vm_public_ip - vm_user = var.admin_username - admin_ssh_public_key = var.ssh_public_key - os_disk_image = var.vm_os_disk_image - resource_group_name = azurerm_resource_group.rg.name - subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] - os_disk_storage_account_type = var.vm_os_disk_storage_account_type - boot_diagnostics_storage_account = module.storage_account.primary_blob_endpoint - log_analytics_workspace_id = module.log_analytics_workspace.workspace_id - log_analytics_workspace_key = module.log_analytics_workspace.primary_shared_key - log_analytics_workspace_resource_id = module.log_analytics_workspace.id - log_analytics_retention_days = var.log_analytics_retention_days - tags = var.tags -} - module "key_vault" { source = "./modules/key_vault" name = var.name_prefix == null ? "${random_string.prefix.result}${var.key_vault_name}" : "${var.name_prefix}${var.key_vault_name}" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/main.tf deleted file mode 100644 index b5169c64c..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/main.tf +++ /dev/null @@ -1,221 +0,0 @@ -resource "azurerm_public_ip" "public_ip" { - name = "${var.name}PublicIp" - location = var.location - resource_group_name = var.resource_group_name - allocation_method = "Dynamic" - domain_name_label = lower(var.name) - count = var.public_ip ? 1 : 0 - tags = var.tags - - lifecycle { - ignore_changes = [ - tags - ] - } -} - -resource "azurerm_network_security_group" "nsg" { - name = "${var.name}Nsg" - location = var.location - resource_group_name = var.resource_group_name - tags = var.tags - - security_rule { - name = "SSH" - priority = 1001 - direction = "Inbound" - access = "Allow" - protocol = "Tcp" - source_port_range = "*" - destination_port_range = "22" - source_address_prefix = "*" - destination_address_prefix = "*" - } - - lifecycle { - ignore_changes = [ - tags - ] - } -} - -resource "azurerm_network_interface" "nic" { - name = "${var.name}Nic" - location = var.location - resource_group_name = var.resource_group_name - tags = var.tags - - ip_configuration { - name = "Configuration" - subnet_id = var.subnet_id - private_ip_address_allocation = "Dynamic" - public_ip_address_id = try(azurerm_public_ip.public_ip[0].id, null) - } - - lifecycle { - ignore_changes = [ - tags - ] - } -} - -resource "azurerm_network_interface_security_group_association" "nsg_association" { - network_interface_id = azurerm_network_interface.nic.id - network_security_group_id = azurerm_network_security_group.nsg.id - depends_on = [azurerm_network_security_group.nsg] -} - -resource "azurerm_linux_virtual_machine" "virtual_machine" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - network_interface_ids = [azurerm_network_interface.nic.id] - size = var.size - computer_name = var.name - admin_username = var.vm_user - tags = var.tags - - os_disk { - name = "${var.name}OsDisk" - caching = "ReadWrite" - storage_account_type = var.os_disk_storage_account_type - } - - admin_ssh_key { - username = var.vm_user - public_key = var.admin_ssh_public_key - } - - source_image_reference { - offer = lookup(var.os_disk_image, "offer", null) - publisher = lookup(var.os_disk_image, "publisher", null) - sku = lookup(var.os_disk_image, "sku", null) - version = lookup(var.os_disk_image, "version", null) - } - - boot_diagnostics { - storage_account_uri = var.boot_diagnostics_storage_account == "" ? null : var.boot_diagnostics_storage_account - } - - lifecycle { - ignore_changes = [ - tags - ] - } - - depends_on = [ - azurerm_network_interface.nic, - azurerm_network_security_group.nsg - ] -} - - -resource "azurerm_virtual_machine_extension" "azure_monitor_agent" { - name = "${var.name}MonitorAgent" - virtual_machine_id = azurerm_linux_virtual_machine.virtual_machine.id - publisher = "Microsoft.Azure.Monitor" - type = "AzureMonitorLinuxAgent" - type_handler_version = "1.21" - auto_upgrade_minor_version = true - automatic_upgrade_enabled = true - tags = var.tags - - lifecycle { - ignore_changes = [ - tags - ] - } -} - -resource "azurerm_monitor_data_collection_rule" "linux" { - name = "LinuxVmMonitorDataCollectionRule" - resource_group_name = var.resource_group_name - location = var.location - tags = var.tags - - destinations { - log_analytics { - workspace_resource_id = var.log_analytics_workspace_resource_id - name = "default" - } - } - - data_flow { - streams = ["Microsoft-InsightsMetrics", "Microsoft-Syslog", "Microsoft-Perf"] - destinations = ["default"] - } - - data_sources { - syslog { - facility_names = ["*"] - log_levels = ["*"] - name = "syslog" - } - - performance_counter { - streams = ["Microsoft-Perf", "Microsoft-InsightsMetrics"] - sampling_frequency_in_seconds = 60 - name = "perfcounter" - counter_specifiers = [ - "\\Processor Information(_Total)\\% Processor Time", - "\\Processor Information(_Total)\\% Privileged Time", - "\\Processor Information(_Total)\\% User Time", - "\\Processor Information(_Total)\\Processor Frequency", - "\\System\\Processes", - "\\Process(_Total)\\Thread Count", - "\\Process(_Total)\\Handle Count", - "\\System\\System Up Time", - "\\System\\Context Switches/sec", - "\\System\\Processor Queue Length", - "\\Memory\\% Committed Bytes In Use", - "\\Memory\\Available Bytes", - "\\Memory\\Committed Bytes", - "\\Memory\\Cache Bytes", - "\\Memory\\Pool Paged Bytes", - "\\Memory\\Pool Nonpaged Bytes", - "\\Memory\\Pages/sec", - "\\Memory\\Page Faults/sec", - "\\Process(_Total)\\Working Set", - "\\Process(_Total)\\Working Set - Private", - "\\LogicalDisk(_Total)\\% Disk Time", - "\\LogicalDisk(_Total)\\% Disk Read Time", - "\\LogicalDisk(_Total)\\% Disk Write Time", - "\\LogicalDisk(_Total)\\% Idle Time", - "\\LogicalDisk(_Total)\\Disk Bytes/sec", - "\\LogicalDisk(_Total)\\Disk Read Bytes/sec", - "\\LogicalDisk(_Total)\\Disk Write Bytes/sec", - "\\LogicalDisk(_Total)\\Disk Transfers/sec", - "\\LogicalDisk(_Total)\\Disk Reads/sec", - "\\LogicalDisk(_Total)\\Disk Writes/sec", - "\\LogicalDisk(_Total)\\Avg. Disk sec/Transfer", - "\\LogicalDisk(_Total)\\Avg. Disk sec/Read", - "\\LogicalDisk(_Total)\\Avg. Disk sec/Write", - "\\LogicalDisk(_Total)\\Avg. Disk Queue Length", - "\\LogicalDisk(_Total)\\Avg. Disk Read Queue Length", - "\\LogicalDisk(_Total)\\Avg. Disk Write Queue Length", - "\\LogicalDisk(_Total)\\% Free Space", - "\\LogicalDisk(_Total)\\Free Megabytes", - "\\Network Interface(*)\\Bytes Total/sec", - "\\Network Interface(*)\\Bytes Sent/sec", - "\\Network Interface(*)\\Bytes Received/sec", - "\\Network Interface(*)\\Packets/sec", - "\\Network Interface(*)\\Packets Sent/sec", - "\\Network Interface(*)\\Packets Received/sec", - "\\Network Interface(*)\\Packets Outbound Errors", - "\\Network Interface(*)\\Packets Received Errors", - ] - } - } - - lifecycle { - ignore_changes = [ - tags - ] - } -} - -resource "azurerm_monitor_data_collection_rule_association" "virtual_machine_association" { - name = "LinuxVmMonitorDataCollectionRuleAssociation" - target_resource_id = azurerm_linux_virtual_machine.virtual_machine.id - data_collection_rule_id = azurerm_monitor_data_collection_rule.linux.id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/outputs.tf deleted file mode 100644 index a7ac8a18b..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/outputs.tf +++ /dev/null @@ -1,9 +0,0 @@ -output "public_ip" { - description = "Specifies the public IP address of the virtual machine" - value = azurerm_linux_virtual_machine.virtual_machine.public_ip_address -} - -output "username" { - description = "Specifies the username of the virtual machine" - value = var.vm_user -} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/variables.tf deleted file mode 100644 index b3a2adbc9..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_machine/variables.tf +++ /dev/null @@ -1,95 +0,0 @@ -variable resource_group_name { - description = "(Required) Specifies the resource group name of the virtual machine" - type = string -} - -variable name { - description = "(Required) Specifies the name of the virtual machine" - type = string -} - -variable size { - description = "(Required) Specifies the size of the virtual machine" - type = string -} - -variable "os_disk_image" { - type = map(string) - description = "(Optional) Specifies the os disk image of the virtual machine" - default = { - publisher = "Canonical" - offer = "0001-com-ubuntu-server-jammy" - sku = "22_04-lts-gen2" - version = "latest" - } -} - -variable "os_disk_storage_account_type" { - description = "(Optional) Specifies the storage account type of the os disk of the virtual machine" - default = "StandardSSD_LRS" - type = string - - validation { - condition = contains(["Premium_LRS", "Premium_ZRS", "StandardSSD_LRS", "StandardSSD_ZRS", "Standard_LRS"], var.os_disk_storage_account_type) - error_message = "The storage account type of the OS disk is invalid." - } -} - -variable public_ip { - description = "(Optional) Specifies whether create a public IP for the virtual machine" - type = bool - default = false -} - -variable location { - description = "(Required) Specifies the location of the virtual machine" - type = string -} - -variable subnet_id { - description = "(Required) Specifies the resource id of the subnet hosting the virtual machine" - type = string -} - -variable vm_user { - description = "(Required) Specifies the username of the virtual machine" - type = string - default = "azadmin" -} - -variable "boot_diagnostics_storage_account" { - description = "(Optional) The Primary/Secondary Endpoint for the Azure Storage Account (general purpose) which should be used to store Boot Diagnostics, including Console Output and Screenshots from the Hypervisor." - default = null -} - -variable "tags" { - description = "(Optional) Specifies the tags of the storage account" - default = {} -} - -variable "log_analytics_workspace_id" { - description = "Specifies the log analytics workspace id" - type = string -} - -variable "log_analytics_workspace_key" { - description = "Specifies the log analytics workspace key" - type = string -} - -variable "log_analytics_workspace_resource_id" { - description = "Specifies the log analytics workspace resource id" - type = string -} - - -variable "log_analytics_retention_days" { - description = "Specifies the number of days of the retention policy" - type = number - default = 7 -} - -variable "admin_ssh_public_key" { - description = "Specifies the public SSH key" - type = string -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 31d28bf69..39ea8bc12 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -356,52 +356,6 @@ variable "user_node_pool_node_count" { default = 3 } -variable "vm_enabled" { - description = "(Optional) Specifies whether create a virtual machine" - type = bool - default = false -} - -variable "vm_name" { - description = "Specifies the name of the jumpbox virtual machine" - default = "Vm" - type = string -} - -variable "vm_public_ip" { - description = "(Optional) Specifies whether create a public IP for the virtual machine" - type = bool - default = false -} - -variable "vm_size" { - description = "Specifies the size of the jumpbox virtual machine" - default = "Standard_DS1_v2" - type = string -} - -variable "vm_os_disk_storage_account_type" { - description = "Specifies the storage account type of the os disk of the jumpbox virtual machine" - default = "Premium_LRS" - type = string - - validation { - condition = contains(["Premium_LRS", "Premium_ZRS", "StandardSSD_LRS", "StandardSSD_ZRS", "Standard_LRS"], var.vm_os_disk_storage_account_type) - error_message = "The storage account type of the OS disk is invalid." - } -} - -variable "vm_os_disk_image" { - type = map(string) - description = "Specifies the os disk image of the virtual machine" - default = { - publisher = "Canonical" - offer = "0001-com-ubuntu-server-jammy" - sku = "22_04-lts-gen2" - version = "latest" - } -} - variable "storage_account_kind" { description = "(Optional) Specifies the account kind of the storage account" default = "StorageV2" @@ -558,11 +512,6 @@ variable "admin_username" { default = "azadmin" } -variable "ssh_public_key" { - description = "(Required) Specifies the SSH public key for the jumpbox virtual machine and AKS worker nodes." - type = string -} - variable "keda_enabled" { description = "(Optional) Specifies whether KEDA Autoscaler can be used for workloads." type = bool From fe6948a5c4dbf40a91edba33cf1923382fccd28e Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 26 Nov 2024 15:51:53 -0500 Subject: [PATCH 007/119] Fix SSH --- .../terraform/.terraform.lock.hcl | 20 ++++++++++++++++ .../AksOpenAiTerraform/terraform/main.tf | 1 - .../terraform/modules/aks/main.tf | 11 ++++++++- .../terraform/modules/aks/outputs.tf | 1 - .../terraform/modules/aks/ssh.tf | 24 +++++++++++++++++++ .../terraform/modules/aks/variables.tf | 5 ---- 6 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl index d3369d32b..52919c9e1 100644 --- a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl +++ b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl @@ -1,6 +1,26 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/azure/azapi" { + version = "2.0.1" + constraints = "~> 2.0.1" + hashes = [ + "h1:VJpm9+TaZ4SC6ncXCiiE+jWmLKZRbrd4KOt79iMIicU=", + "zh:3df16ed604be5f4ccd5d52a02c2681d8eb2f5a4462625c983cb17c20cdf0bfb2", + "zh:4efd9961ea52990e21385086f0b3324edfb534ea6a8f0f6ba146a74bfb56aa63", + "zh:5561418efc9744c9873855a146226608778e29b4c0c3b3872634ef2da2d86593", + "zh:7ebcb4c6ca71c87850df67d4e5f79ce4a036d4131b8c11ae0b9b8787353843b8", + "zh:81a9259cb1e45507e9431794fbd354dd4d8b78c6a9508b0bfa108b00e6ad23cb", + "zh:8c1836fa186272347f97c7a3884556979618d1b93721e8a24203d90ff4efbd40", + "zh:a72bdd43a11a383525764720d24cb78ec5d9f1167f129d05448108fef1ba7af3", + "zh:ade9d17c6b8717e7b04af5a9d1a948d047ac4dcf6affb2485afa3ad0a2eaee15", + "zh:b3c5bfcab98251cb0c157dbe78dc6d0864c9bf364d316003c84c1e624a3c3524", + "zh:c33b872a2473a9b052add89e4557d361b0ebaa42865e99b95465050d2c858d43", + "zh:efe425f8ecd4d79448214c93ef10881b3b74cf2d9b5211d76f05aced22621eb4", + "zh:ff704c5e73e832507367d9d962b6b53c0ca3c724689f0974feffd5339c3db18a", + ] +} + provider "registry.terraform.io/hashicorp/azurerm" { version = "3.58.0" constraints = "3.58.0" diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 421ed68e7..cfe012598 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -160,7 +160,6 @@ module "aks_cluster" { admin_group_object_ids = var.admin_group_object_ids azure_rbac_enabled = var.azure_rbac_enabled admin_username = var.admin_username - ssh_public_key = var.ssh_public_key keda_enabled = var.keda_enabled vertical_pod_autoscaler_enabled = var.vertical_pod_autoscaler_enabled workload_identity_enabled = var.workload_identity_enabled diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index 49a6622a6..04807b6b3 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -1,3 +1,12 @@ +terraform { + required_providers { + azapi = { + source = "Azure/azapi" + version = "~>2.0.1" + } + } +} + resource "azurerm_user_assigned_identity" "aks_identity" { resource_group_name = var.resource_group_name location = var.location @@ -50,7 +59,7 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { linux_profile { admin_username = var.admin_username ssh_key { - key_data = var.ssh_public_key + key_data = azapi_resource_action.ssh_public_key_gen.output.publicKey } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf index 576a7399d..fd2e362d8 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf @@ -8,7 +8,6 @@ output "id" { description = "Specifies the resource id of the AKS cluster." } - output "aks_identity_principal_id" { value = azurerm_user_assigned_identity.aks_identity.principal_id description = "Specifies the principal id of the managed identity of the AKS cluster." diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf new file mode 100644 index 000000000..4cb7b3c37 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf @@ -0,0 +1,24 @@ +resource "random_pet" "ssh_key_name" { + prefix = "ssh" + separator = "" +} + +resource "azapi_resource_action" "ssh_public_key_gen" { + type = "Microsoft.Compute/sshPublicKeys@2024-07-01" + resource_id = azapi_resource.ssh_public_key.id + action = "generateKeyPair" + method = "POST" + + response_export_values = ["publicKey", "privateKey"] +} + +resource "azapi_resource" "ssh_public_key" { + type = "Microsoft.Compute/sshPublicKeys@2024-07-01" + name = random_pet.ssh_key_name.id + location = var.location + parent_id = var.resource_group_id +} + +output "key_data" { + value = azapi_resource_action.ssh_public_key_gen.output.publicKey +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf index 33c66482b..c4518c28c 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf @@ -262,11 +262,6 @@ variable "admin_username" { default = "azadmin" } -variable "ssh_public_key" { - description = "(Required) Specifies the SSH public key used to access the cluster. Changing this forces a new resource to be created." - type = string -} - variable "keda_enabled" { description = "(Optional) Specifies whether KEDA Autoscaler can be used for workloads." type = bool From 21ca3f674840c3698c9460c86b0aea05d67269ac Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 26 Nov 2024 16:47:46 -0500 Subject: [PATCH 008/119] Update provider --- .../terraform/.terraform.lock.hcl | 30 +++++++++---------- .../AksOpenAiTerraform/terraform/main.tf | 2 +- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl index 52919c9e1..9df9eb753 100644 --- a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl +++ b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl @@ -22,30 +22,28 @@ provider "registry.terraform.io/azure/azapi" { } provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.58.0" - constraints = "3.58.0" + version = "4.11.0" + constraints = "4.11.0" hashes = [ - "h1:Hvlt3hgTiip6xMeq8/EDGqF8NoVuZjYdTZdO79YNXsw=", - "h1:ceZlVBDs02TjOxY4JGLaeqCigsy7gcEPLcJudiTurb4=", - "zh:22b19802605ca3e2b811e33650438be3647748cf8f75474c78448c30ac1cad0b", - "zh:402ce010f4b68337abaccf8059c37294cabcbdbc3cefd9491dcd312e36ceea3c", - "zh:53d2cd15f1631c7ffb47918064d644899cc671d47c72f4dafee4e2a5e69afd14", - "zh:5a6b1c55629cff555472d1d43ad6e802693f7fd046c7d37718d4de6f52dbf66b", - "zh:6181dccb7bca7cd84b0295a0332f19a7347a9586101f0a5e51b53bda1ec74651", - "zh:854181d6a8821b3707775c913e91dd7944fcb55098953ef030168fa3cd0224aa", - "zh:b44c758424d1a037fd833e0c69b29e3ac4047ab95653bb3e080835e55bd9badb", - "zh:b6ee916a1579bba29b1aacce8897c6733fa97ba0dba2808f1ffa9ab492743fab", - "zh:b7ab57044649578410dadfdf4412fc5f8aa085a25ea0b061393e843b49b43b63", - "zh:cb68ddb922eb4be74dedf58c953d7f778b4e5f3cdcbe2ea83e02b12296ce4969", + "h1:l1igOrMmeHJHXEj9eLkx9Uiq/iKKbukoRuPUIDGBY/8=", + "zh:026808a5ff8bce161518d503bfc57c4a95637d67e923a94382c8e878c96aaf00", + "zh:13473ebb56ed701fdd8c288a220cef3ec6ee170fb1ac45c6ce5a612848e64690", + "zh:36667374d31509456fd928f651fc1ccc7438c53bc99cf9ec3b6ec6e7f791394e", + "zh:5f44e16aab36a93391ce81b9a93b694fecf11f71615f2414ee40bb5e211d3dbb", + "zh:9310e860f9236d0f7171e05444ca85e239f0938b9fb08ec3bfd9712a14013308", + "zh:aaf6ea1f68526a175e84424710b06dd6cf8987b404206cc581692560c1530810", + "zh:b6d1965af0aed85f3eccaaec5dae90f59632bf07e2bf5b7473359a7c761872a5", + "zh:c642675ea2d8e1f1bb440016238ab25fa4270cb155b01e90598161488df47128", + "zh:d22d07834c2a5da6ce7054699d4f708277fccb63436cfbf6c90c58cddddba408", + "zh:eceb91d652ea9145531129c7da50603e9415812f639acbf1720d51f878798fb8", + "zh:f26bf55ce68c1ed6e316ee70652bc3cc357987ea4b3caf6f835405850c6897e0", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:fe9e86173134cd9dc8ed65eae8634abc6d6f6806b5b412f54fccf4047052daa0", ] } provider "registry.terraform.io/hashicorp/random" { version = "3.6.3" hashes = [ - "h1:Fnaec9vA8sZ8BXVlN3Xn9Jz3zghSETIKg7ch8oXhxno=", "h1:N2IQabOiZC5eCEGrfgVS6ChVmRDh1ENtfHgGjnV4QQQ=", "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8", diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index cfe012598..8a61f3e18 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "3.58" + version = "4.11.0" } } } From 7e072c40146c86a2b0a2d295100fbd9a850f253f Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 27 Nov 2024 08:48:47 -0500 Subject: [PATCH 009/119] Fix Breaking changes --- scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf | 4 ++-- .../terraform/modules/storage_account/main.tf | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf index 8af163a57..707fe31fb 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf @@ -31,8 +31,8 @@ resource "azurerm_cognitive_deployment" "deployment" { version = each.value.model.version } - scale { - type = "Standard" + sku { + name = "Standard" } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf index fdaccb7bd..2cfa39239 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf @@ -9,6 +9,8 @@ resource "azurerm_storage_account" "storage_account" { is_hns_enabled = var.is_hns_enabled tags = var.tags + allow_nested_items_to_be_public = false + network_rules { default_action = (length(var.ip_rules) + length(var.virtual_network_subnet_ids)) > 0 ? "Deny" : var.default_action ip_rules = var.ip_rules From 0e56a214510c6f05ff56b0e28f90f658ea3838ef Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 27 Nov 2024 08:35:16 -0500 Subject: [PATCH 010/119] Create README template --- scenarios/AksOpenAiTerraform/README.md | 35 +++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index b046df346..0d8378ae4 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -8,11 +8,40 @@ ms.author: ariaamini ms.custom: innovation-engine, linux-related-content --- + + ## Install AKS extension Run commands below to set up AKS extensions for Azure. ```bash -# ./terraform/register-preview-features.sh -echo "HI" -``` \ No newline at end of file +./terraform/register-preview-features.sh +``` + +## Set up service principal + +A Service Principal is an application within Azure Active Directory with the authentication tokens Terraform needs to perform actions on your behalf. + +```bash +# TODO: fix +# az ad sp create-for-rbac --role="Contributor" --scopes="/subscriptions/$ARM_SUBSCRIPTION_ID" +``` + +## Setup Infra + +```bash +export ARM_SUBSCRIPTION_ID="0c8875c7-e423-4caa-827a-1f0350bd8dd3" +# For debugging in powershell +# $env:ARM_SUBSCRIPTION_ID = "0c8875c7-e423-4caa-827a-1f0350bd8dd3" + +terraform apply +``` + +## Set up environment + +```bash +export ARM_CLIENT_ID="" +export ARM_CLIENT_SECRET="" +export ARM_SUBSCRIPTION_ID="" +export ARM_TENANT_ID="" +``` From 35128aa1e6337d1cb5731f14d80043a5c85a246b Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 27 Nov 2024 08:43:47 -0500 Subject: [PATCH 011/119] Fix logs --- .../AksOpenAiTerraform/terraform/main.tf | 2 - .../terraform/modules/aks/main.tf | 42 +----------------- .../terraform/modules/bastion_host/main.tf | 34 +------------- .../modules/container_registry/main.tf | 17 +------ .../modules/container_registry/variables.tf | 6 --- .../modules/diagnostic_setting/main.tf | 10 ----- .../terraform/modules/firewall/main.tf | 44 +------------------ .../terraform/modules/key_vault/main.tf | 17 +------ .../modules/network_security_group/main.tf | 13 +----- .../terraform/modules/openai/main.tf | 22 +--------- .../terraform/modules/virtual_network/main.tf | 12 +---- .../modules/virtual_network/variables.tf | 6 --- 12 files changed, 10 insertions(+), 215 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 8a61f3e18..a3b58b43e 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -55,7 +55,6 @@ module "virtual_network" { vnet_name = var.name_prefix == null ? "${random_string.prefix.result}${var.vnet_name}" : "${var.name_prefix}${var.vnet_name}" address_space = var.vnet_address_space log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = var.log_analytics_retention_days tags = var.tags subnets = [ @@ -118,7 +117,6 @@ module "container_registry" { admin_enabled = var.acr_admin_enabled georeplication_locations = var.acr_georeplication_locations log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = var.log_analytics_retention_days tags = var.tags } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index 04807b6b3..3e8ca3c9d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -111,79 +111,39 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { } resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "DiagnosticsSettings" + name = "AksDiagnosticsSettings" target_resource_id = azurerm_kubernetes_cluster.aks_cluster.id log_analytics_workspace_id = var.log_analytics_workspace_id enabled_log { category = "kube-apiserver" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "kube-audit" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "kube-audit-admin" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "kube-controller-manager" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "kube-scheduler" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "cluster-autoscaler" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "guard" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } metric { category = "AllMetrics" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf index 34a01d40b..cbc9428cd 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf @@ -33,67 +33,37 @@ resource "azurerm_bastion_host" "bastion_host" { } resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "DiagnosticsSettings" + name = "BastionDiagnosticsSettings" target_resource_id = azurerm_bastion_host.bastion_host.id log_analytics_workspace_id = var.log_analytics_workspace_id enabled_log { category = "BastionAuditLogs" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } metric { category = "AllMetrics" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } } resource "azurerm_monitor_diagnostic_setting" "pip_settings" { - name = "DiagnosticsSettings" + name = "BastionDdosDiagnosticsSettings" target_resource_id = azurerm_public_ip.public_ip.id log_analytics_workspace_id = var.log_analytics_workspace_id enabled_log { category = "DDoSProtectionNotifications" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "DDoSMitigationFlowLogs" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "DDoSMitigationReports" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } metric { category = "AllMetrics" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf index 38e3b49f3..44a9a669c 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf @@ -44,34 +44,19 @@ resource "azurerm_user_assigned_identity" "acr_identity" { } resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "DiagnosticsSettings" + name = "ContainerDiagnosticsSettings" target_resource_id = azurerm_container_registry.acr.id log_analytics_workspace_id = var.log_analytics_workspace_id enabled_log { category = "ContainerRegistryRepositoryEvents" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "ContainerRegistryLoginEvents" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } metric { category = "AllMetrics" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf index 3bf6ae317..6550f9570 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf @@ -45,10 +45,4 @@ variable "georeplication_locations" { variable "log_analytics_workspace_id" { description = "Specifies the log analytics workspace id" type = string -} - -variable "log_analytics_retention_days" { - description = "Specifies the number of days of the retention policy" - type = number - default = 7 } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf index 4456c789f..45d29f614 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf @@ -15,11 +15,6 @@ resource "azurerm_monitor_diagnostic_setting" "settings" { content { category = each.key enabled = true - - retention_policy { - enabled = var.retention_policy_enabled - days = var.retention_policy_days - } } } @@ -28,11 +23,6 @@ resource "azurerm_monitor_diagnostic_setting" "settings" { content { category = each.key enabled = true - - retention_policy { - enabled = var.retention_policy_enabled - days = var.retention_policy_days - } } } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf index 3f535454b..479cefb33 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf @@ -226,85 +226,45 @@ resource "azurerm_firewall_policy_rule_collection_group" "policy" { } resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "DiagnosticsSettings" + name = "FirewallDiagnosticsSettings" target_resource_id = azurerm_firewall.firewall.id log_analytics_workspace_id = var.log_analytics_workspace_id enabled_log { category = "AzureFirewallApplicationRule" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "AzureFirewallNetworkRule" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "AzureFirewallDnsProxy" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } metric { category = "AllMetrics" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } } resource "azurerm_monitor_diagnostic_setting" "pip_settings" { - name = "DiagnosticsSettings" + name = "FirewallDdosDiagnosticsSettings" target_resource_id = azurerm_public_ip.pip.id log_analytics_workspace_id = var.log_analytics_workspace_id enabled_log { category = "DDoSProtectionNotifications" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "DDoSMitigationFlowLogs" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "DDoSMitigationReports" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } metric { category = "AllMetrics" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf index df166f775..0f3f899b6 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf @@ -31,34 +31,19 @@ resource "azurerm_key_vault" "key_vault" { } resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "DiagnosticsSettings" + name = "KeyVaultDiagnosticsSettings" target_resource_id = azurerm_key_vault.key_vault.id log_analytics_workspace_id = var.log_analytics_workspace_id enabled_log { category = "AuditEvent" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "AzurePolicyEvaluationDetails" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } metric { category = "AllMetrics" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf index 80edbd556..b1a7589cb 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf @@ -33,26 +33,15 @@ resource "azurerm_network_security_group" "nsg" { } resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "DiagnosticsSettings" + name = "NetworkSecurityDiagnosticsSettings" target_resource_id = azurerm_network_security_group.nsg.id log_analytics_workspace_id = var.log_analytics_workspace_id enabled_log { category = "NetworkSecurityGroupEvent" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "NetworkSecurityGroupRuleCounter" - enabled = true - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf index 707fe31fb..55d6d49c7 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf @@ -37,43 +37,23 @@ resource "azurerm_cognitive_deployment" "deployment" { } resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "DiagnosticsSettings" + name = "OpenAiDiagnosticsSettings" target_resource_id = azurerm_cognitive_account.openai.id log_analytics_workspace_id = var.log_analytics_workspace_id enabled_log { category = "Audit" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "RequestResponse" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } enabled_log { category = "Trace" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } metric { category = "AllMetrics" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf index cf3cd7bcd..02b2ecd00 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf @@ -35,25 +35,15 @@ resource "azurerm_subnet" "subnet" { } resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "DiagnosticsSettings" + name = "VirtualNetworkDiagnosticsSettings" target_resource_id = azurerm_virtual_network.vnet.id log_analytics_workspace_id = var.log_analytics_workspace_id enabled_log { category = "VMProtectionAlerts" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } metric { category = "AllMetrics" - - retention_policy { - enabled = true - days = var.log_analytics_retention_days - } } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf index 6252c0c1b..e65ed517a 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf @@ -37,10 +37,4 @@ variable "tags" { variable "log_analytics_workspace_id" { description = "Specifies the log analytics workspace id" type = string -} - -variable "log_analytics_retention_days" { - description = "Specifies the number of days of the retention policy" - type = number - default = 7 } \ No newline at end of file From 978325f920f21dbf3fe742e411543964541f7e95 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 27 Nov 2024 08:45:20 -0500 Subject: [PATCH 012/119] Update region --- scenarios/AksOpenAiTerraform/terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 39ea8bc12..b2fb93cee 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -30,7 +30,7 @@ variable "solution_plan_map" { variable "location" { description = "Specifies the location for the resource group and all the resources" - default = "northeurope" + default = "eastus" type = string } From 5e9d1655fdf7f4a854ad5f4c63c0f838097c86eb Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 27 Nov 2024 08:47:09 -0500 Subject: [PATCH 013/119] Fix vnet --- .../AksOpenAiTerraform/terraform/main.tf | 21 +++++++++++++------ .../terraform/modules/virtual_network/main.tf | 5 +++-- .../modules/virtual_network/variables.tf | 7 +++++-- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index a3b58b43e..02d7cdf2f 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -61,35 +61,41 @@ module "virtual_network" { { name : var.system_node_pool_subnet_name address_prefixes : var.system_node_pool_subnet_address_prefix - private_endpoint_network_policies_enabled : true + private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false delegation: null }, { name : var.user_node_pool_subnet_name address_prefixes : var.user_node_pool_subnet_address_prefix - private_endpoint_network_policies_enabled : true + private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false delegation: null }, { name : var.pod_subnet_name address_prefixes : var.pod_subnet_address_prefix - private_endpoint_network_policies_enabled : true + private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false - delegation: "Microsoft.ContainerService/managedClusters" + delegation = { + name = "delegation" + service_delegation = { + name = "Microsoft.ContainerService/managedClusters" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } }, { name : var.vm_subnet_name address_prefixes : var.vm_subnet_address_prefix - private_endpoint_network_policies_enabled : true + private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false delegation: null }, { name : "AzureBastionSubnet" address_prefixes : var.bastion_subnet_address_prefix - private_endpoint_network_policies_enabled : true + private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false delegation: null } @@ -106,6 +112,9 @@ module "nat_gateway" { zones = var.nat_gateway_zones tags = var.tags subnet_ids = module.virtual_network.subnet_ids + depends_on = [ + module.virtual_network + ] } module "container_registry" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf index 02b2ecd00..68dfb2c4a 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf @@ -19,7 +19,7 @@ resource "azurerm_subnet" "subnet" { resource_group_name = var.resource_group_name virtual_network_name = azurerm_virtual_network.vnet.name address_prefixes = each.value.address_prefixes - private_endpoint_network_policies_enabled = each.value.private_endpoint_network_policies_enabled + private_endpoint_network_policies = each.value.private_endpoint_network_policies private_link_service_network_policies_enabled = each.value.private_link_service_network_policies_enabled dynamic "delegation" { @@ -28,7 +28,8 @@ resource "azurerm_subnet" "subnet" { name = "delegation" service_delegation { - name = delegation.value + name = delegation.value.service_delegation.name + actions = delegation.value.service_delegation.actions } } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf index e65ed517a..02dec85dd 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf @@ -23,9 +23,12 @@ variable "subnets" { type = list(object({ name = string address_prefixes = list(string) - private_endpoint_network_policies_enabled = bool + private_endpoint_network_policies = string private_link_service_network_policies_enabled = bool - delegation = string + delegation = object({name = string, service_delegation = object({ + name = string + actions = list(string) + })}) })) } From 3057190e9542c7b90d9cd551692c1357af31f5b1 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 27 Nov 2024 08:49:01 -0500 Subject: [PATCH 014/119] Fix aks --- .../AksOpenAiTerraform/terraform/main.tf | 12 --- .../terraform/modules/aks/main.tf | 12 +-- .../terraform/modules/aks/variables.tf | 54 -------------- .../terraform/modules/node_pool/main.tf | 6 -- .../terraform/modules/node_pool/variables.tf | 27 +------ .../AksOpenAiTerraform/terraform/variables.tf | 73 +------------------ 6 files changed, 5 insertions(+), 179 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 02d7cdf2f..567258ce0 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -139,7 +139,6 @@ module "aks_cluster" { kubernetes_version = var.kubernetes_version dns_prefix = lower(var.aks_cluster_name) private_cluster_enabled = var.private_cluster_enabled - automatic_channel_upgrade = var.automatic_channel_upgrade sku_tier = var.sku_tier system_node_pool_name = var.system_node_pool_name system_node_pool_vm_size = var.system_node_pool_vm_size @@ -147,14 +146,7 @@ module "aks_cluster" { pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] system_node_pool_availability_zones = var.system_node_pool_availability_zones system_node_pool_node_labels = var.system_node_pool_node_labels - system_node_pool_node_taints = var.system_node_pool_node_taints - system_node_pool_enable_auto_scaling = var.system_node_pool_enable_auto_scaling - system_node_pool_enable_host_encryption = var.system_node_pool_enable_host_encryption - system_node_pool_enable_node_public_ip = var.system_node_pool_enable_node_public_ip system_node_pool_max_pods = var.system_node_pool_max_pods - system_node_pool_max_count = var.system_node_pool_max_count - system_node_pool_min_count = var.system_node_pool_min_count - system_node_pool_node_count = var.system_node_pool_node_count system_node_pool_os_disk_type = var.system_node_pool_os_disk_type tags = var.tags network_dns_service_ip = var.network_dns_service_ip @@ -194,14 +186,10 @@ module "node_pool" { availability_zones = var.user_node_pool_availability_zones vnet_subnet_id = module.virtual_network.subnet_ids[var.user_node_pool_subnet_name] pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] - enable_auto_scaling = var.user_node_pool_enable_auto_scaling enable_host_encryption = var.user_node_pool_enable_host_encryption enable_node_public_ip = var.user_node_pool_enable_node_public_ip orchestrator_version = var.kubernetes_version max_pods = var.user_node_pool_max_pods - max_count = var.user_node_pool_max_count - min_count = var.user_node_pool_min_count - node_count = var.user_node_pool_node_count os_type = var.user_node_pool_os_type priority = var.user_node_pool_priority tags = var.tags diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index 3e8ca3c9d..6178c43f3 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -28,30 +28,25 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { kubernetes_version = var.kubernetes_version dns_prefix = var.dns_prefix private_cluster_enabled = var.private_cluster_enabled - automatic_channel_upgrade = var.automatic_channel_upgrade + automatic_upgrade_channel = "stable" sku_tier = var.sku_tier workload_identity_enabled = var.workload_identity_enabled oidc_issuer_enabled = var.oidc_issuer_enabled open_service_mesh_enabled = var.open_service_mesh_enabled image_cleaner_enabled = var.image_cleaner_enabled azure_policy_enabled = var.azure_policy_enabled + image_cleaner_interval_hours = 72 http_application_routing_enabled = var.http_application_routing_enabled default_node_pool { name = var.system_node_pool_name + node_count = 1 vm_size = var.system_node_pool_vm_size vnet_subnet_id = var.vnet_subnet_id pod_subnet_id = var.pod_subnet_id zones = var.system_node_pool_availability_zones node_labels = var.system_node_pool_node_labels - node_taints = var.system_node_pool_node_taints - enable_auto_scaling = var.system_node_pool_enable_auto_scaling - enable_host_encryption = var.system_node_pool_enable_host_encryption - enable_node_public_ip = var.system_node_pool_enable_node_public_ip max_pods = var.system_node_pool_max_pods - max_count = var.system_node_pool_max_count - min_count = var.system_node_pool_min_count - node_count = var.system_node_pool_node_count os_disk_type = var.system_node_pool_os_disk_type tags = var.tags } @@ -91,7 +86,6 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { } azure_active_directory_role_based_access_control { - managed = true tenant_id = var.tenant_id admin_group_object_ids = var.admin_group_object_ids azure_rbac_enabled = var.azure_rbac_enabled diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf index c4518c28c..ebcb393bb 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf @@ -47,17 +47,6 @@ variable "role_based_access_control_enabled" { type = bool } -variable "automatic_channel_upgrade" { - description = "(Optional) The upgrade channel for this Kubernetes Cluster. Possible values are patch, rapid, and stable." - default = "stable" - type = string - - validation { - condition = contains( ["patch", "rapid", "stable"], var.automatic_channel_upgrade) - error_message = "The upgrade mode is invalid." - } -} - variable "sku_tier" { description = "(Optional) The SKU Tier that should be used for this Kubernetes Cluster. Possible values are Free and Paid (which includes the Uptime SLA). Defaults to Free." default = "Free" @@ -71,7 +60,6 @@ variable "sku_tier" { variable "kubernetes_version" { description = "Specifies the AKS Kubernetes version" - default = "1.21.1" type = string } @@ -134,24 +122,6 @@ variable "system_node_pool_subnet_address_prefix" { type = list(string) } -variable "system_node_pool_enable_auto_scaling" { - description = "(Optional) Whether to enable auto-scaler. Defaults to false." - type = bool - default = true -} - -variable "system_node_pool_enable_host_encryption" { - description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." - type = bool - default = false -} - -variable "system_node_pool_enable_node_public_ip" { - description = "(Optional) Should each node have a Public IP Address? Defaults to false. Changing this forces a new resource to be created." - type = bool - default = false -} - variable "system_node_pool_max_pods" { description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." type = number @@ -164,36 +134,12 @@ variable "system_node_pool_node_labels" { default = {} } -variable "system_node_pool_node_taints" { - description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." - type = list(string) - default = [] -} - variable "system_node_pool_os_disk_type" { description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." type = string default = "Ephemeral" } -variable "system_node_pool_max_count" { - description = "(Required) The maximum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be greater than or equal to min_count." - type = number - default = 10 -} - -variable "system_node_pool_min_count" { - description = "(Required) The minimum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be less than or equal to max_count." - type = number - default = 3 -} - -variable "system_node_pool_node_count" { - description = "(Optional) The initial number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be a value in the range min_count - max_count." - type = number - default = 3 -} - variable "log_analytics_workspace_id" { description = "(Optional) The ID of the Log Analytics Workspace which the OMS Agent should send data to. Must be present if enabled is true." type = string diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf index e13f1340b..acdeda9c3 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf @@ -8,15 +8,9 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { zones = var.availability_zones vnet_subnet_id = var.vnet_subnet_id pod_subnet_id = var.pod_subnet_id - enable_auto_scaling = var.enable_auto_scaling - enable_host_encryption = var.enable_host_encryption - enable_node_public_ip = var.enable_node_public_ip proximity_placement_group_id = var.proximity_placement_group_id orchestrator_version = var.orchestrator_version max_pods = var.max_pods - max_count = var.max_count - min_count = var.min_count - node_count = var.node_count os_disk_size_gb = var.os_disk_size_gb os_disk_type = var.os_disk_type os_type = var.os_type diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf index 688b179b8..b95bf813f 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf @@ -19,12 +19,6 @@ variable "availability_zones" { default = ["1", "2", "3"] } -variable "enable_auto_scaling" { - description = "(Optional) Whether to enable auto-scaler. Defaults to false." - type = bool - default = false -} - variable "enable_host_encryption" { description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." type = bool @@ -67,9 +61,8 @@ variable "tags" { } variable "orchestrator_version" { - description = "(Optional) Version of Kubernetes used for the Agents. If not specified, the latest recommended version will be used at provisioning time (but won't auto-upgrade)" + description = "(Required) Version of Kubernetes used for the Agents. If not specified, the latest recommended version will be used at provisioning time (but won't auto-upgrade)" type = string - default = null } variable "os_disk_size_gb" { @@ -114,24 +107,6 @@ variable "pod_subnet_id" { default = null } -variable "max_count" { - description = "(Required) The maximum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be greater than or equal to min_count." - type = number - default = 10 -} - -variable "min_count" { - description = "(Required) The minimum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be less than or equal to max_count." - type = number - default = 3 -} - -variable "node_count" { - description = "(Optional) The initial number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be a value in the range min_count - max_count." - type = number - default = 3 -} - variable resource_group_name { description = "Specifies the resource group name" type = string diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index b2fb93cee..ebb8e7e29 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -124,17 +124,6 @@ variable "role_based_access_control_enabled" { type = bool } -variable "automatic_channel_upgrade" { - description = "(Optional) The upgrade channel for this Kubernetes Cluster. Possible values are patch, rapid, and stable." - default = "stable" - type = string - - validation { - condition = contains( ["patch", "rapid", "stable"], var.automatic_channel_upgrade) - error_message = "The upgrade mode is invalid." - } -} - variable "admin_group_object_ids" { description = "(Optional) A list of Object IDs of Azure Active Directory Groups which should have Admin Role on the Cluster." default = [] @@ -160,7 +149,7 @@ variable "sku_tier" { variable "kubernetes_version" { description = "Specifies the AKS Kubernetes version" - default = "1.26.3" + default = "1.31.1" type = string } @@ -200,24 +189,6 @@ variable "system_node_pool_name" { type = string } -variable "system_node_pool_enable_auto_scaling" { - description = "(Optional) Whether to enable auto-scaler. Defaults to false." - type = bool - default = true -} - -variable "system_node_pool_enable_host_encryption" { - description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." - type = bool - default = false -} - -variable "system_node_pool_enable_node_public_ip" { - description = "(Optional) Should each node have a Public IP Address? Defaults to false. Changing this forces a new resource to be created." - type = bool - default = false -} - variable "system_node_pool_max_pods" { description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." type = number @@ -242,24 +213,6 @@ variable "system_node_pool_os_disk_type" { default = "Ephemeral" } -variable "system_node_pool_max_count" { - description = "(Required) The maximum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be greater than or equal to min_count." - type = number - default = 10 -} - -variable "system_node_pool_min_count" { - description = "(Required) The minimum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be less than or equal to max_count." - type = number - default = 3 -} - -variable "system_node_pool_node_count" { - description = "(Optional) The initial number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be a value in the range min_count - max_count." - type = number - default = 3 -} - variable "user_node_pool_name" { description = "(Required) Specifies the name of the node pool." type = string @@ -278,12 +231,6 @@ variable "user_node_pool_availability_zones" { default = ["1", "2", "3"] } -variable "user_node_pool_enable_auto_scaling" { - description = "(Optional) Whether to enable auto-scaler. Defaults to false." - type = bool - default = true -} - variable "user_node_pool_enable_host_encryption" { description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." type = bool @@ -338,24 +285,6 @@ variable "user_node_pool_priority" { default = "Regular" } -variable "user_node_pool_max_count" { - description = "(Required) The maximum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be greater than or equal to min_count." - type = number - default = 10 -} - -variable "user_node_pool_min_count" { - description = "(Required) The minimum number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be less than or equal to max_count." - type = number - default = 3 -} - -variable "user_node_pool_node_count" { - description = "(Optional) The initial number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be a value in the range min_count - max_count." - type = number - default = 3 -} - variable "storage_account_kind" { description = "(Optional) Specifies the account kind of the storage account" default = "StorageV2" From 1045fd9ceaa6381adf81d3afdf94213e1bf9961a Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Wed, 27 Nov 2024 08:53:57 -0500 Subject: [PATCH 015/119] Remove depends on --- scenarios/AksOpenAiTerraform/terraform/main.tf | 3 --- 1 file changed, 3 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 567258ce0..a34d87607 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -112,9 +112,6 @@ module "nat_gateway" { zones = var.nat_gateway_zones tags = var.tags subnet_ids = module.virtual_network.subnet_ids - depends_on = [ - module.virtual_network - ] } module "container_registry" { From 99153c193457a33878fd5775a5dc9397b6017dd8 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Wed, 27 Nov 2024 09:09:07 -0500 Subject: [PATCH 016/119] Rename + small fix --- .../terraform/modules/log_analytics/variables.tf | 3 +-- scenarios/AksOpenAiTerraform/terraform/variables.tf | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf index 107a0a8da..d6226a996 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf @@ -25,9 +25,8 @@ variable "sku" { } variable "solution_plan_map" { - description = "(Optional) Specifies the map structure containing the list of solutions to be enabled." + description = "(Required) Specifies the map structure containing the list of solutions to be enabled." type = map(any) - default = {} } variable "tags" { diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index ebb8e7e29..76eb314c9 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -1,7 +1,7 @@ variable "name_prefix" { description = "(Optional) A prefix for the name of all the resource groups and resources." type = string - default = "Bingo" + default = "BingoTest" nullable = true } From ff91a32ad10bacfc585f06bb4f4f4c1fc87e9085 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 27 Nov 2024 10:17:55 -0500 Subject: [PATCH 017/119] Remove log --- .../terraform/modules/virtual_network/main.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf index 68dfb2c4a..bb9443977 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf @@ -40,10 +40,6 @@ resource "azurerm_monitor_diagnostic_setting" "settings" { target_resource_id = azurerm_virtual_network.vnet.id log_analytics_workspace_id = var.log_analytics_workspace_id - enabled_log { - category = "VMProtectionAlerts" - } - metric { category = "AllMetrics" } From 8bac6341807438bf3ff3064d5afd0e18d82492e3 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Wed, 27 Nov 2024 10:18:23 -0500 Subject: [PATCH 018/119] Remove default var --- scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf index ebcb393bb..a054a87a0 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf @@ -65,7 +65,6 @@ variable "kubernetes_version" { variable "system_node_pool_vm_size" { description = "Specifies the vm size of the system node pool" - default = "Standard_F8s_v2" type = string } From 5ad62571b42a2d6be4ee065bff71c3f64e55b900 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Wed, 27 Nov 2024 10:36:52 -0500 Subject: [PATCH 019/119] Update SKU --- scenarios/AksOpenAiTerraform/terraform/variables.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 76eb314c9..b29052c1f 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -30,7 +30,7 @@ variable "solution_plan_map" { variable "location" { description = "Specifies the location for the resource group and all the resources" - default = "eastus" + default = "westus2" type = string } @@ -149,13 +149,13 @@ variable "sku_tier" { variable "kubernetes_version" { description = "Specifies the AKS Kubernetes version" - default = "1.31.1" + default = "1.29.10" type = string } variable "system_node_pool_vm_size" { description = "Specifies the vm size of the system node pool" - default = "Standard_F8s_v2" + default = "Standard_D8ds_v5" type = string } @@ -222,7 +222,7 @@ variable "user_node_pool_name" { variable "user_node_pool_vm_size" { description = "(Required) The SKU which should be used for the Virtual Machines used in this Node Pool. Changing this forces a new resource to be created." type = string - default = "Standard_F8s_v2" + default = "Standard_D8ds_v5" } variable "user_node_pool_availability_zones" { From 7e1e26edcc618293517c48b64db3fa2910f86f36 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Wed, 27 Nov 2024 10:56:40 -0500 Subject: [PATCH 020/119] Change name again --- scenarios/AksOpenAiTerraform/terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index b29052c1f..c783a6c94 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -1,7 +1,7 @@ variable "name_prefix" { description = "(Optional) A prefix for the name of all the resource groups and resources." type = string - default = "BingoTest" + default = "BingoTestName" nullable = true } From 988e6a0ca38c7edf82185070de5174907340ef8e Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 12:21:19 -0800 Subject: [PATCH 021/119] Fixes --- scenarios/AksOpenAiTerraform/terraform/main.tf | 2 +- scenarios/AksOpenAiTerraform/terraform/outputs.tf | 0 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/outputs.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index a34d87607..7ea14b035 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "4.11.0" + version = "~> 4.16.0" } } } diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf deleted file mode 100644 index e69de29bb..000000000 From afccc7adbc131ad24eb0cf21a2bf0a193ae09e06 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 12:44:48 -0800 Subject: [PATCH 022/119] Inline + remove tags --- .../AksOpenAiTerraform/terraform/main.tf | 644 +++++++++++++++++- .../AksOpenAiTerraform/terraform/variables.tf | 621 ----------------- 2 files changed, 622 insertions(+), 643 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 7ea14b035..1790ea998 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -33,10 +33,631 @@ resource "random_string" "storage_account_suffix" { numeric = false } +variable "name_prefix" { + description = "(Optional) A prefix for the name of all the resource groups and resources." + type = string + default = "BingoTestName" + nullable = true +} + +variable "log_analytics_workspace_name" { + description = "Specifies the name of the log analytics workspace" + default = "Workspace" + type = string +} + +variable "log_analytics_retention_days" { + description = "Specifies the number of days of the retention policy" + type = number + default = 30 +} + +variable "solution_plan_map" { + description = "Specifies solutions to deploy to log analytics workspace" + default = { + ContainerInsights= { + product = "OMSGallery/ContainerInsights" + publisher = "Microsoft" + } + } + type = map(any) +} + +variable "location" { + description = "Specifies the location for the resource group and all the resources" + default = "westus2" + type = string +} + +variable "resource_group_name" { + description = "Specifies the resource group name" + default = "RG" + type = string +} + +variable "vnet_name" { + description = "Specifies the name of the AKS subnet" + default = "AksVNet" + type = string +} + +variable "vnet_address_space" { + description = "Specifies the address prefix of the AKS subnet" + default = ["10.0.0.0/8"] + type = list(string) +} + +variable "system_node_pool_subnet_name" { + description = "Specifies the name of the subnet that hosts the system node pool" + default = "SystemSubnet" + type = string +} + +variable "system_node_pool_subnet_address_prefix" { + description = "Specifies the address prefix of the subnet that hosts the system node pool" + default = ["10.240.0.0/16"] + type = list(string) +} + +variable "user_node_pool_subnet_name" { + description = "Specifies the name of the subnet that hosts the user node pool" + default = "UserSubnet" + type = string +} + +variable "user_node_pool_subnet_address_prefix" { + description = "Specifies the address prefix of the subnet that hosts the user node pool" + type = list(string) + default = ["10.241.0.0/16"] +} + +variable "pod_subnet_name" { + description = "Specifies the name of the jumpbox subnet" + default = "PodSubnet" + type = string +} + +variable "pod_subnet_address_prefix" { + description = "Specifies the address prefix of the jumbox subnet" + default = ["10.242.0.0/16"] + type = list(string) +} + +variable "vm_subnet_name" { + description = "Specifies the name of the jumpbox subnet" + default = "VmSubnet" + type = string +} + +variable "vm_subnet_address_prefix" { + description = "Specifies the address prefix of the jumbox subnet" + default = ["10.243.1.0/24"] + type = list(string) +} + +variable "bastion_subnet_address_prefix" { + description = "Specifies the address prefix of the firewall subnet" + default = ["10.243.2.0/24"] + type = list(string) +} + +variable "aks_cluster_name" { + description = "(Required) Specifies the name of the AKS cluster." + default = "Aks" + type = string +} + +variable "private_cluster_enabled" { + description = "(Optional) Specifies wether the AKS cluster be private or not." + default = false + type = bool +} + +variable "role_based_access_control_enabled" { + description = "(Required) Is Role Based Access Control Enabled? Changing this forces a new resource to be created." + default = true + type = bool +} + +variable "admin_group_object_ids" { + description = "(Optional) A list of Object IDs of Azure Active Directory Groups which should have Admin Role on the Cluster." + default = [] + type = list(string) +} + +variable "azure_rbac_enabled" { + description = "(Optional) Is Role Based Access Control based on Azure AD enabled?" + default = true + type = bool +} + +variable "sku_tier" { + description = "(Optional) The SKU Tier that should be used for this Kubernetes Cluster. Possible values are Free and Paid (which includes the Uptime SLA). Defaults to Free." + default = "Free" + type = string + + validation { + condition = contains( ["Free", "Paid"], var.sku_tier) + error_message = "The sku tier is invalid." + } +} + +variable "kubernetes_version" { + description = "Specifies the AKS Kubernetes version" + default = "1.29.10" + type = string +} + +variable "system_node_pool_vm_size" { + description = "Specifies the vm size of the system node pool" + default = "Standard_D8ds_v5" + type = string +} + +variable "system_node_pool_availability_zones" { + description = "Specifies the availability zones of the system node pool" + default = ["1", "2", "3"] + type = list(string) +} + +variable "network_dns_service_ip" { + description = "Specifies the DNS service IP" + default = "10.2.0.10" + type = string +} + +variable "network_service_cidr" { + description = "Specifies the service CIDR" + default = "10.2.0.0/24" + type = string +} + +variable "network_plugin" { + description = "Specifies the network plugin of the AKS cluster" + default = "azure" + type = string +} + +variable "system_node_pool_name" { + description = "Specifies the name of the system node pool" + default = "system" + type = string +} + +variable "system_node_pool_max_pods" { + description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." + type = number + default = 50 +} + +variable "system_node_pool_node_labels" { + description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." + type = map(any) + default = {} +} + +variable "system_node_pool_node_taints" { + description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." + type = list(string) + default = ["CriticalAddonsOnly=true:NoSchedule"] +} + +variable "system_node_pool_os_disk_type" { + description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." + type = string + default = "Ephemeral" +} + +variable "user_node_pool_name" { + description = "(Required) Specifies the name of the node pool." + type = string + default = "user" +} + +variable "user_node_pool_vm_size" { + description = "(Required) The SKU which should be used for the Virtual Machines used in this Node Pool. Changing this forces a new resource to be created." + type = string + default = "Standard_D8ds_v5" +} + +variable "user_node_pool_availability_zones" { + description = "(Optional) A list of Availability Zones where the Nodes in this Node Pool should be created in. Changing this forces a new resource to be created." + type = list(string) + default = ["1", "2", "3"] +} + +variable "user_node_pool_enable_host_encryption" { + description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." + type = bool + default = false +} + +variable "user_node_pool_enable_node_public_ip" { + description = "(Optional) Should each node have a Public IP Address? Defaults to false. Changing this forces a new resource to be created." + type = bool + default = false +} + +variable "user_node_pool_max_pods" { + description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." + type = number + default = 50 +} + +variable "user_node_pool_mode" { + description = "(Optional) Should this Node Pool be used for System or User resources? Possible values are System and User. Defaults to User." + type = string + default = "User" +} + +variable "user_node_pool_node_labels" { + description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." + type = map(any) + default = {} +} + +variable "user_node_pool_node_taints" { + description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." + type = list(string) + default = [] +} + +variable "user_node_pool_os_disk_type" { + description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." + type = string + default = "Ephemeral" +} + +variable "user_node_pool_os_type" { + description = "(Optional) The Operating System which should be used for this Node Pool. Changing this forces a new resource to be created. Possible values are Linux and Windows. Defaults to Linux." + type = string + default = "Linux" +} + +variable "user_node_pool_priority" { + description = "(Optional) The Priority for Virtual Machines within the Virtual Machine Scale Set that powers this Node Pool. Possible values are Regular and Spot. Defaults to Regular. Changing this forces a new resource to be created." + type = string + default = "Regular" +} + +variable "storage_account_kind" { + description = "(Optional) Specifies the account kind of the storage account" + default = "StorageV2" + type = string + + validation { + condition = contains(["Storage", "StorageV2"], var.storage_account_kind) + error_message = "The account kind of the storage account is invalid." + } +} + +variable "storage_account_tier" { + description = "(Optional) Specifies the account tier of the storage account" + default = "Standard" + type = string + + validation { + condition = contains(["Standard", "Premium"], var.storage_account_tier) + error_message = "The account tier of the storage account is invalid." + } +} + +variable "acr_name" { + description = "Specifies the name of the container registry" + type = string + default = "Acr" +} + +variable "acr_sku" { + description = "Specifies the name of the container registry" + type = string + default = "Premium" + + validation { + condition = contains(["Basic", "Standard", "Premium"], var.acr_sku) + error_message = "The container registry sku is invalid." + } +} + +variable "acr_admin_enabled" { + description = "Specifies whether admin is enabled for the container registry" + type = bool + default = true +} + +variable "acr_georeplication_locations" { + description = "(Optional) A list of Azure locations where the container registry should be geo-replicated." + type = list(string) + default = [] +} + +variable "tags" { + description = "(Optional) Specifies tags for all the resources" + default = { + createdWith = "Terraform" + } +} + +variable "bastion_host_name" { + description = "(Optional) Specifies the name of the bastion host" + default = "BastionHost" + type = string +} + +variable "storage_account_replication_type" { + description = "(Optional) Specifies the replication type of the storage account" + default = "LRS" + type = string + + validation { + condition = contains(["LRS", "ZRS", "GRS", "GZRS", "RA-GRS", "RA-GZRS"], var.storage_account_replication_type) + error_message = "The replication type of the storage account is invalid." + } +} + +variable "key_vault_name" { + description = "Specifies the name of the key vault." + type = string + default = "KeyVault" +} + +variable "key_vault_sku_name" { + description = "(Required) The Name of the SKU used for this Key Vault. Possible values are standard and premium." + type = string + default = "standard" + + validation { + condition = contains(["standard", "premium" ], var.key_vault_sku_name) + error_message = "The sku name of the key vault is invalid." + } +} + +variable"key_vault_enabled_for_deployment" { + description = "(Optional) Boolean flag to specify whether Azure Virtual Machines are permitted to retrieve certificates stored as secrets from the key vault. Defaults to false." + type = bool + default = true +} + +variable"key_vault_enabled_for_disk_encryption" { + description = " (Optional) Boolean flag to specify whether Azure Disk Encryption is permitted to retrieve secrets from the vault and unwrap keys. Defaults to false." + type = bool + default = true +} + +variable"key_vault_enabled_for_template_deployment" { + description = "(Optional) Boolean flag to specify whether Azure Resource Manager is permitted to retrieve secrets from the key vault. Defaults to false." + type = bool + default = true +} + +variable"key_vault_enable_rbac_authorization" { + description = "(Optional) Boolean flag to specify whether Azure Key Vault uses Role Based Access Control (RBAC) for authorization of data actions. Defaults to false." + type = bool + default = true +} + +variable"key_vault_purge_protection_enabled" { + description = "(Optional) Is Purge Protection enabled for this Key Vault? Defaults to false." + type = bool + default = false +} + +variable "key_vault_soft_delete_retention_days" { + description = "(Optional) The number of days that items should be retained for once soft-deleted. This value can be between 7 and 90 (the default) days." + type = number + default = 30 +} + +variable "key_vault_bypass" { + description = "(Required) Specifies which traffic can bypass the network rules. Possible values are AzureServices and None." + type = string + default = "AzureServices" + + validation { + condition = contains(["AzureServices", "None" ], var.key_vault_bypass) + error_message = "The valut of the bypass property of the key vault is invalid." + } +} + +variable "key_vault_default_action" { + description = "(Required) The Default Action to use when no rules match from ip_rules / virtual_network_subnet_ids. Possible values are Allow and Deny." + type = string + default = "Allow" + + validation { + condition = contains(["Allow", "Deny" ], var.key_vault_default_action) + error_message = "The value of the default action property of the key vault is invalid." + } +} + +variable "admin_username" { + description = "(Required) Specifies the admin username of the jumpbox virtual machine and AKS worker nodes." + type = string + default = "azadmin" +} + +variable "keda_enabled" { + description = "(Optional) Specifies whether KEDA Autoscaler can be used for workloads." + type = bool + default = true +} + +variable "vertical_pod_autoscaler_enabled" { + description = "(Optional) Specifies whether Vertical Pod Autoscaler should be enabled." + type = bool + default = true +} + +variable "workload_identity_enabled" { + description = "(Optional) Specifies whether Azure AD Workload Identity should be enabled for the Cluster. Defaults to false." + type = bool + default = true +} + +variable "oidc_issuer_enabled" { + description = "(Optional) Enable or Disable the OIDC issuer URL." + type = bool + default = true +} + +variable "open_service_mesh_enabled" { + description = "(Optional) Is Open Service Mesh enabled? For more details, please visit Open Service Mesh for AKS." + type = bool + default = true +} + +variable "image_cleaner_enabled" { + description = "(Optional) Specifies whether Image Cleaner is enabled." + type = bool + default = true +} + +variable "azure_policy_enabled" { + description = "(Optional) Should the Azure Policy Add-On be enabled? For more details please visit Understand Azure Policy for Azure Kubernetes Service" + type = bool + default = true +} + +variable "http_application_routing_enabled" { + description = "(Optional) Should HTTP Application Routing be enabled?" + type = bool + default = false +} + +variable "openai_name" { + description = "(Required) Specifies the name of the Azure OpenAI Service" + type = string + default = "OpenAi" +} + +variable "openai_sku_name" { + description = "(Optional) Specifies the sku name for the Azure OpenAI Service" + type = string + default = "S0" +} + +variable "openai_custom_subdomain_name" { + description = "(Optional) Specifies the custom subdomain name of the Azure OpenAI Service" + type = string + nullable = true + default = "" +} + +variable "openai_public_network_access_enabled" { + description = "(Optional) Specifies whether public network access is allowed for the Azure OpenAI Service" + type = bool + default = true +} + +variable "openai_deployments" { + description = "(Optional) Specifies the deployments of the Azure OpenAI Service" + type = list(object({ + name = string + model = object({ + name = string + version = string + }) + rai_policy_name = string + })) + default = [ + { + name = "gpt-35-turbo" + model = { + name = "gpt-35-turbo" + version = "0301" + } + rai_policy_name = "" + } + ] +} + +variable "nat_gateway_name" { + description = "(Required) Specifies the name of the Azure OpenAI Service" + type = string + default = "NatGateway" +} + +variable "nat_gateway_sku_name" { + description = "(Optional) The SKU which should be used. At this time the only supported value is Standard. Defaults to Standard" + type = string + default = "Standard" +} + +variable "nat_gateway_idle_timeout_in_minutes" { + description = "(Optional) The idle timeout which should be used in minutes. Defaults to 4." + type = number + default = 4 +} + +variable "nat_gateway_zones" { + description = " (Optional) A list of Availability Zones in which this NAT Gateway should be located. Changing this forces a new NAT Gateway to be created." + type = list(string) + default = ["1"] +} + +variable "workload_managed_identity_name" { + description = "(Required) Specifies the name of the workload user-defined managed identity." + type = string + default = "WorkloadManagedIdentity" +} + +variable "subdomain" { + description = "Specifies the subdomain of the Kubernetes ingress object." + type = string + default = "magic8ball" +} + +variable "domain" { + description = "Specifies the domain of the Kubernetes ingress object." + type = string + default = "contoso.com" +} + +variable "namespace" { + description = "Specifies the namespace of the workload application that accesses the Azure OpenAI Service." + type = string + default = "magic8ball" +} + +variable "service_account_name" { + description = "Specifies the name of the service account of the workload application that accesses the Azure OpenAI Service." + type = string + default = "magic8ball-sa" +} + +variable "email" { + description = "Specifies the email address for the cert-manager cluster issuer." + type = string + default = "paolos@microsoft.com" +} + +variable "deployment_script_name" { + description = "(Required) Specifies the name of the Azure OpenAI Service" + type = string + default = "BashScript" +} + +variable "deployment_script_azure_cli_version" { + description = "(Required) Azure CLI module version to be used." + type = string + default = "2.9.1" +} + +variable "deployment_script_managed_identity_name" { + description = "Specifies the name of the user-defined managed identity used by the deployment script." + type = string + default = "ScriptManagedIdentity" +} + +variable "deployment_script_primary_script_uri" { + description = "(Optional) Uri for the script. This is the entry point for the external script. Changing this forces a new Resource Deployment Script to be created." + type = string + default = "https://paolosalvatori.blob.core.windows.net/scripts/install-nginx-via-helm-and-create-sa.sh" +} + resource "azurerm_resource_group" "rg" { name = var.name_prefix == null ? "${random_string.prefix.result}${var.resource_group_name}" : "${var.name_prefix}${var.resource_group_name}" location = var.location - tags = var.tags } module "log_analytics_workspace" { @@ -45,7 +666,6 @@ module "log_analytics_workspace" { location = var.location resource_group_name = azurerm_resource_group.rg.name solution_plan_map = var.solution_plan_map - tags = var.tags } module "virtual_network" { @@ -55,7 +675,6 @@ module "virtual_network" { vnet_name = var.name_prefix == null ? "${random_string.prefix.result}${var.vnet_name}" : "${var.name_prefix}${var.vnet_name}" address_space = var.vnet_address_space log_analytics_workspace_id = module.log_analytics_workspace.id - tags = var.tags subnets = [ { @@ -110,7 +729,6 @@ module "nat_gateway" { sku_name = var.nat_gateway_sku_name idle_timeout_in_minutes = var.nat_gateway_idle_timeout_in_minutes zones = var.nat_gateway_zones - tags = var.tags subnet_ids = module.virtual_network.subnet_ids } @@ -123,8 +741,6 @@ module "container_registry" { admin_enabled = var.acr_admin_enabled georeplication_locations = var.acr_georeplication_locations log_analytics_workspace_id = module.log_analytics_workspace.id - tags = var.tags - } module "aks_cluster" { @@ -145,7 +761,6 @@ module "aks_cluster" { system_node_pool_node_labels = var.system_node_pool_node_labels system_node_pool_max_pods = var.system_node_pool_max_pods system_node_pool_os_disk_type = var.system_node_pool_os_disk_type - tags = var.tags network_dns_service_ip = var.network_dns_service_ip network_plugin = var.network_plugin outbound_type = "userAssignedNATGateway" @@ -189,7 +804,6 @@ module "node_pool" { max_pods = var.user_node_pool_max_pods os_type = var.user_node_pool_os_type priority = var.user_node_pool_priority - tags = var.tags } module "openai" { @@ -198,7 +812,6 @@ module "openai" { location = var.location resource_group_name = azurerm_resource_group.rg.name sku_name = var.openai_sku_name - tags = var.tags deployments = var.openai_deployments custom_subdomain_name = var.openai_custom_subdomain_name == "" || var.openai_custom_subdomain_name == null ? var.name_prefix == null ? lower("${random_string.prefix.result}${var.openai_name}") : lower("${var.name_prefix}${var.openai_name}") : lower(var.openai_custom_subdomain_name) public_network_access_enabled = var.openai_public_network_access_enabled @@ -210,7 +823,6 @@ resource "azurerm_user_assigned_identity" "aks_workload_identity" { name = var.name_prefix == null ? "${random_string.prefix.result}${var.workload_managed_identity_name}" : "${var.name_prefix}${var.workload_managed_identity_name}" resource_group_name = azurerm_resource_group.rg.name location = var.location - tags = var.tags lifecycle { ignore_changes = [ @@ -257,8 +869,6 @@ module "storage_account" { account_kind = var.storage_account_kind account_tier = var.storage_account_tier replication_type = var.storage_account_replication_type - tags = var.tags - } module "bastion_host" { @@ -289,14 +899,12 @@ module "key_vault" { default_action = var.key_vault_default_action log_analytics_workspace_id = module.log_analytics_workspace.id log_analytics_retention_days = var.log_analytics_retention_days - tags = var.tags } module "acr_private_dns_zone" { source = "./modules/private_dns_zone" name = "privatelink.azurecr.io" resource_group_name = azurerm_resource_group.rg.name - tags = var.tags virtual_networks_to_link = { (module.virtual_network.name) = { subscription_id = data.azurerm_client_config.current.subscription_id @@ -309,7 +917,6 @@ module "openai_private_dns_zone" { source = "./modules/private_dns_zone" name = "privatelink.openai.azure.com" resource_group_name = azurerm_resource_group.rg.name - tags = var.tags virtual_networks_to_link = { (module.virtual_network.name) = { subscription_id = data.azurerm_client_config.current.subscription_id @@ -322,7 +929,6 @@ module "key_vault_private_dns_zone" { source = "./modules/private_dns_zone" name = "privatelink.vaultcore.azure.net" resource_group_name = azurerm_resource_group.rg.name - tags = var.tags virtual_networks_to_link = { (module.virtual_network.name) = { subscription_id = data.azurerm_client_config.current.subscription_id @@ -335,7 +941,6 @@ module "blob_private_dns_zone" { source = "./modules/private_dns_zone" name = "privatelink.blob.core.windows.net" resource_group_name = azurerm_resource_group.rg.name - tags = var.tags virtual_networks_to_link = { (module.virtual_network.name) = { subscription_id = data.azurerm_client_config.current.subscription_id @@ -350,7 +955,6 @@ module "openai_private_endpoint" { location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] - tags = var.tags private_connection_resource_id = module.openai.id is_manual_connection = false subresource_name = "account" @@ -364,7 +968,6 @@ module "acr_private_endpoint" { location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] - tags = var.tags private_connection_resource_id = module.container_registry.id is_manual_connection = false subresource_name = "registry" @@ -378,7 +981,6 @@ module "key_vault_private_endpoint" { location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] - tags = var.tags private_connection_resource_id = module.key_vault.id is_manual_connection = false subresource_name = "vault" @@ -392,7 +994,6 @@ module "blob_private_endpoint" { location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] - tags = var.tags private_connection_resource_id = module.storage_account.id is_manual_connection = false subresource_name = "blob" @@ -416,7 +1017,6 @@ module "deployment_script" { tenant_id = data.azurerm_client_config.current.tenant_id subscription_id = data.azurerm_client_config.current.subscription_id workload_managed_identity_client_id = azurerm_user_assigned_identity.aks_workload_identity.client_id - tags = var.tags depends_on = [ module.aks_cluster diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index c783a6c94..e69de29bb 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -1,621 +0,0 @@ -variable "name_prefix" { - description = "(Optional) A prefix for the name of all the resource groups and resources." - type = string - default = "BingoTestName" - nullable = true -} - -variable "log_analytics_workspace_name" { - description = "Specifies the name of the log analytics workspace" - default = "Workspace" - type = string -} - -variable "log_analytics_retention_days" { - description = "Specifies the number of days of the retention policy" - type = number - default = 30 -} - -variable "solution_plan_map" { - description = "Specifies solutions to deploy to log analytics workspace" - default = { - ContainerInsights= { - product = "OMSGallery/ContainerInsights" - publisher = "Microsoft" - } - } - type = map(any) -} - -variable "location" { - description = "Specifies the location for the resource group and all the resources" - default = "westus2" - type = string -} - -variable "resource_group_name" { - description = "Specifies the resource group name" - default = "RG" - type = string -} - -variable "vnet_name" { - description = "Specifies the name of the AKS subnet" - default = "AksVNet" - type = string -} - -variable "vnet_address_space" { - description = "Specifies the address prefix of the AKS subnet" - default = ["10.0.0.0/8"] - type = list(string) -} - -variable "system_node_pool_subnet_name" { - description = "Specifies the name of the subnet that hosts the system node pool" - default = "SystemSubnet" - type = string -} - -variable "system_node_pool_subnet_address_prefix" { - description = "Specifies the address prefix of the subnet that hosts the system node pool" - default = ["10.240.0.0/16"] - type = list(string) -} - -variable "user_node_pool_subnet_name" { - description = "Specifies the name of the subnet that hosts the user node pool" - default = "UserSubnet" - type = string -} - -variable "user_node_pool_subnet_address_prefix" { - description = "Specifies the address prefix of the subnet that hosts the user node pool" - type = list(string) - default = ["10.241.0.0/16"] -} - -variable "pod_subnet_name" { - description = "Specifies the name of the jumpbox subnet" - default = "PodSubnet" - type = string -} - -variable "pod_subnet_address_prefix" { - description = "Specifies the address prefix of the jumbox subnet" - default = ["10.242.0.0/16"] - type = list(string) -} - -variable "vm_subnet_name" { - description = "Specifies the name of the jumpbox subnet" - default = "VmSubnet" - type = string -} - -variable "vm_subnet_address_prefix" { - description = "Specifies the address prefix of the jumbox subnet" - default = ["10.243.1.0/24"] - type = list(string) -} - -variable "bastion_subnet_address_prefix" { - description = "Specifies the address prefix of the firewall subnet" - default = ["10.243.2.0/24"] - type = list(string) -} - -variable "aks_cluster_name" { - description = "(Required) Specifies the name of the AKS cluster." - default = "Aks" - type = string -} - -variable "private_cluster_enabled" { - description = "(Optional) Specifies wether the AKS cluster be private or not." - default = false - type = bool -} - -variable "role_based_access_control_enabled" { - description = "(Required) Is Role Based Access Control Enabled? Changing this forces a new resource to be created." - default = true - type = bool -} - -variable "admin_group_object_ids" { - description = "(Optional) A list of Object IDs of Azure Active Directory Groups which should have Admin Role on the Cluster." - default = [] - type = list(string) -} - -variable "azure_rbac_enabled" { - description = "(Optional) Is Role Based Access Control based on Azure AD enabled?" - default = true - type = bool -} - -variable "sku_tier" { - description = "(Optional) The SKU Tier that should be used for this Kubernetes Cluster. Possible values are Free and Paid (which includes the Uptime SLA). Defaults to Free." - default = "Free" - type = string - - validation { - condition = contains( ["Free", "Paid"], var.sku_tier) - error_message = "The sku tier is invalid." - } -} - -variable "kubernetes_version" { - description = "Specifies the AKS Kubernetes version" - default = "1.29.10" - type = string -} - -variable "system_node_pool_vm_size" { - description = "Specifies the vm size of the system node pool" - default = "Standard_D8ds_v5" - type = string -} - -variable "system_node_pool_availability_zones" { - description = "Specifies the availability zones of the system node pool" - default = ["1", "2", "3"] - type = list(string) -} - -variable "network_dns_service_ip" { - description = "Specifies the DNS service IP" - default = "10.2.0.10" - type = string -} - -variable "network_service_cidr" { - description = "Specifies the service CIDR" - default = "10.2.0.0/24" - type = string -} - -variable "network_plugin" { - description = "Specifies the network plugin of the AKS cluster" - default = "azure" - type = string -} - -variable "system_node_pool_name" { - description = "Specifies the name of the system node pool" - default = "system" - type = string -} - -variable "system_node_pool_max_pods" { - description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." - type = number - default = 50 -} - -variable "system_node_pool_node_labels" { - description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." - type = map(any) - default = {} -} - -variable "system_node_pool_node_taints" { - description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." - type = list(string) - default = ["CriticalAddonsOnly=true:NoSchedule"] -} - -variable "system_node_pool_os_disk_type" { - description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." - type = string - default = "Ephemeral" -} - -variable "user_node_pool_name" { - description = "(Required) Specifies the name of the node pool." - type = string - default = "user" -} - -variable "user_node_pool_vm_size" { - description = "(Required) The SKU which should be used for the Virtual Machines used in this Node Pool. Changing this forces a new resource to be created." - type = string - default = "Standard_D8ds_v5" -} - -variable "user_node_pool_availability_zones" { - description = "(Optional) A list of Availability Zones where the Nodes in this Node Pool should be created in. Changing this forces a new resource to be created." - type = list(string) - default = ["1", "2", "3"] -} - -variable "user_node_pool_enable_host_encryption" { - description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." - type = bool - default = false -} - -variable "user_node_pool_enable_node_public_ip" { - description = "(Optional) Should each node have a Public IP Address? Defaults to false. Changing this forces a new resource to be created." - type = bool - default = false -} - -variable "user_node_pool_max_pods" { - description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." - type = number - default = 50 -} - -variable "user_node_pool_mode" { - description = "(Optional) Should this Node Pool be used for System or User resources? Possible values are System and User. Defaults to User." - type = string - default = "User" -} - -variable "user_node_pool_node_labels" { - description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." - type = map(any) - default = {} -} - -variable "user_node_pool_node_taints" { - description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." - type = list(string) - default = [] -} - -variable "user_node_pool_os_disk_type" { - description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." - type = string - default = "Ephemeral" -} - -variable "user_node_pool_os_type" { - description = "(Optional) The Operating System which should be used for this Node Pool. Changing this forces a new resource to be created. Possible values are Linux and Windows. Defaults to Linux." - type = string - default = "Linux" -} - -variable "user_node_pool_priority" { - description = "(Optional) The Priority for Virtual Machines within the Virtual Machine Scale Set that powers this Node Pool. Possible values are Regular and Spot. Defaults to Regular. Changing this forces a new resource to be created." - type = string - default = "Regular" -} - -variable "storage_account_kind" { - description = "(Optional) Specifies the account kind of the storage account" - default = "StorageV2" - type = string - - validation { - condition = contains(["Storage", "StorageV2"], var.storage_account_kind) - error_message = "The account kind of the storage account is invalid." - } -} - -variable "storage_account_tier" { - description = "(Optional) Specifies the account tier of the storage account" - default = "Standard" - type = string - - validation { - condition = contains(["Standard", "Premium"], var.storage_account_tier) - error_message = "The account tier of the storage account is invalid." - } -} - -variable "acr_name" { - description = "Specifies the name of the container registry" - type = string - default = "Acr" -} - -variable "acr_sku" { - description = "Specifies the name of the container registry" - type = string - default = "Premium" - - validation { - condition = contains(["Basic", "Standard", "Premium"], var.acr_sku) - error_message = "The container registry sku is invalid." - } -} - -variable "acr_admin_enabled" { - description = "Specifies whether admin is enabled for the container registry" - type = bool - default = true -} - -variable "acr_georeplication_locations" { - description = "(Optional) A list of Azure locations where the container registry should be geo-replicated." - type = list(string) - default = [] -} - -variable "tags" { - description = "(Optional) Specifies tags for all the resources" - default = { - createdWith = "Terraform" - } -} - -variable "bastion_host_name" { - description = "(Optional) Specifies the name of the bastion host" - default = "BastionHost" - type = string -} - -variable "storage_account_replication_type" { - description = "(Optional) Specifies the replication type of the storage account" - default = "LRS" - type = string - - validation { - condition = contains(["LRS", "ZRS", "GRS", "GZRS", "RA-GRS", "RA-GZRS"], var.storage_account_replication_type) - error_message = "The replication type of the storage account is invalid." - } -} - -variable "key_vault_name" { - description = "Specifies the name of the key vault." - type = string - default = "KeyVault" -} - -variable "key_vault_sku_name" { - description = "(Required) The Name of the SKU used for this Key Vault. Possible values are standard and premium." - type = string - default = "standard" - - validation { - condition = contains(["standard", "premium" ], var.key_vault_sku_name) - error_message = "The sku name of the key vault is invalid." - } -} - -variable"key_vault_enabled_for_deployment" { - description = "(Optional) Boolean flag to specify whether Azure Virtual Machines are permitted to retrieve certificates stored as secrets from the key vault. Defaults to false." - type = bool - default = true -} - -variable"key_vault_enabled_for_disk_encryption" { - description = " (Optional) Boolean flag to specify whether Azure Disk Encryption is permitted to retrieve secrets from the vault and unwrap keys. Defaults to false." - type = bool - default = true -} - -variable"key_vault_enabled_for_template_deployment" { - description = "(Optional) Boolean flag to specify whether Azure Resource Manager is permitted to retrieve secrets from the key vault. Defaults to false." - type = bool - default = true -} - -variable"key_vault_enable_rbac_authorization" { - description = "(Optional) Boolean flag to specify whether Azure Key Vault uses Role Based Access Control (RBAC) for authorization of data actions. Defaults to false." - type = bool - default = true -} - -variable"key_vault_purge_protection_enabled" { - description = "(Optional) Is Purge Protection enabled for this Key Vault? Defaults to false." - type = bool - default = false -} - -variable "key_vault_soft_delete_retention_days" { - description = "(Optional) The number of days that items should be retained for once soft-deleted. This value can be between 7 and 90 (the default) days." - type = number - default = 30 -} - -variable "key_vault_bypass" { - description = "(Required) Specifies which traffic can bypass the network rules. Possible values are AzureServices and None." - type = string - default = "AzureServices" - - validation { - condition = contains(["AzureServices", "None" ], var.key_vault_bypass) - error_message = "The valut of the bypass property of the key vault is invalid." - } -} - -variable "key_vault_default_action" { - description = "(Required) The Default Action to use when no rules match from ip_rules / virtual_network_subnet_ids. Possible values are Allow and Deny." - type = string - default = "Allow" - - validation { - condition = contains(["Allow", "Deny" ], var.key_vault_default_action) - error_message = "The value of the default action property of the key vault is invalid." - } -} - -variable "admin_username" { - description = "(Required) Specifies the admin username of the jumpbox virtual machine and AKS worker nodes." - type = string - default = "azadmin" -} - -variable "keda_enabled" { - description = "(Optional) Specifies whether KEDA Autoscaler can be used for workloads." - type = bool - default = true -} - -variable "vertical_pod_autoscaler_enabled" { - description = "(Optional) Specifies whether Vertical Pod Autoscaler should be enabled." - type = bool - default = true -} - -variable "workload_identity_enabled" { - description = "(Optional) Specifies whether Azure AD Workload Identity should be enabled for the Cluster. Defaults to false." - type = bool - default = true -} - -variable "oidc_issuer_enabled" { - description = "(Optional) Enable or Disable the OIDC issuer URL." - type = bool - default = true -} - -variable "open_service_mesh_enabled" { - description = "(Optional) Is Open Service Mesh enabled? For more details, please visit Open Service Mesh for AKS." - type = bool - default = true -} - -variable "image_cleaner_enabled" { - description = "(Optional) Specifies whether Image Cleaner is enabled." - type = bool - default = true -} - -variable "azure_policy_enabled" { - description = "(Optional) Should the Azure Policy Add-On be enabled? For more details please visit Understand Azure Policy for Azure Kubernetes Service" - type = bool - default = true -} - -variable "http_application_routing_enabled" { - description = "(Optional) Should HTTP Application Routing be enabled?" - type = bool - default = false -} - -variable "openai_name" { - description = "(Required) Specifies the name of the Azure OpenAI Service" - type = string - default = "OpenAi" -} - -variable "openai_sku_name" { - description = "(Optional) Specifies the sku name for the Azure OpenAI Service" - type = string - default = "S0" -} - -variable "openai_custom_subdomain_name" { - description = "(Optional) Specifies the custom subdomain name of the Azure OpenAI Service" - type = string - nullable = true - default = "" -} - -variable "openai_public_network_access_enabled" { - description = "(Optional) Specifies whether public network access is allowed for the Azure OpenAI Service" - type = bool - default = true -} - -variable "openai_deployments" { - description = "(Optional) Specifies the deployments of the Azure OpenAI Service" - type = list(object({ - name = string - model = object({ - name = string - version = string - }) - rai_policy_name = string - })) - default = [ - { - name = "gpt-35-turbo" - model = { - name = "gpt-35-turbo" - version = "0301" - } - rai_policy_name = "" - } - ] -} - -variable "nat_gateway_name" { - description = "(Required) Specifies the name of the Azure OpenAI Service" - type = string - default = "NatGateway" -} - -variable "nat_gateway_sku_name" { - description = "(Optional) The SKU which should be used. At this time the only supported value is Standard. Defaults to Standard" - type = string - default = "Standard" -} - -variable "nat_gateway_idle_timeout_in_minutes" { - description = "(Optional) The idle timeout which should be used in minutes. Defaults to 4." - type = number - default = 4 -} - -variable "nat_gateway_zones" { - description = " (Optional) A list of Availability Zones in which this NAT Gateway should be located. Changing this forces a new NAT Gateway to be created." - type = list(string) - default = ["1"] -} - -variable "workload_managed_identity_name" { - description = "(Required) Specifies the name of the workload user-defined managed identity." - type = string - default = "WorkloadManagedIdentity" -} - -variable "subdomain" { - description = "Specifies the subdomain of the Kubernetes ingress object." - type = string - default = "magic8ball" -} - -variable "domain" { - description = "Specifies the domain of the Kubernetes ingress object." - type = string - default = "contoso.com" -} - -variable "namespace" { - description = "Specifies the namespace of the workload application that accesses the Azure OpenAI Service." - type = string - default = "magic8ball" -} - -variable "service_account_name" { - description = "Specifies the name of the service account of the workload application that accesses the Azure OpenAI Service." - type = string - default = "magic8ball-sa" -} - -variable "email" { - description = "Specifies the email address for the cert-manager cluster issuer." - type = string - default = "paolos@microsoft.com" -} - -variable "deployment_script_name" { - description = "(Required) Specifies the name of the Azure OpenAI Service" - type = string - default = "BashScript" -} - -variable "deployment_script_azure_cli_version" { - description = "(Required) Azure CLI module version to be used." - type = string - default = "2.9.1" -} - -variable "deployment_script_managed_identity_name" { - description = "Specifies the name of the user-defined managed identity used by the deployment script." - type = string - default = "ScriptManagedIdentity" -} - -variable "deployment_script_primary_script_uri" { - description = "(Optional) Uri for the script. This is the entry point for the external script. Changing this forces a new Resource Deployment Script to be created." - type = string - default = "https://paolosalvatori.blob.core.windows.net/scripts/install-nginx-via-helm-and-create-sa.sh" -} \ No newline at end of file From 643431ee189a0428c8471aab90083cb2b284c087 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 13:25:54 -0800 Subject: [PATCH 023/119] Inline a bunch --- .../AksOpenAiTerraform/terraform/main.tf | 331 ++---------------- 1 file changed, 38 insertions(+), 293 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 1790ea998..971960f18 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -34,10 +34,8 @@ resource "random_string" "storage_account_suffix" { } variable "name_prefix" { - description = "(Optional) A prefix for the name of all the resource groups and resources." + description = "A prefix for the name of all the resource groups and resources." type = string - default = "BingoTestName" - nullable = true } variable "log_analytics_workspace_name" { @@ -52,17 +50,6 @@ variable "log_analytics_retention_days" { default = 30 } -variable "solution_plan_map" { - description = "Specifies solutions to deploy to log analytics workspace" - default = { - ContainerInsights= { - product = "OMSGallery/ContainerInsights" - publisher = "Microsoft" - } - } - type = map(any) -} - variable "location" { description = "Specifies the location for the resource group and all the resources" default = "westus2" @@ -75,113 +62,36 @@ variable "resource_group_name" { type = string } -variable "vnet_name" { - description = "Specifies the name of the AKS subnet" - default = "AksVNet" - type = string -} - -variable "vnet_address_space" { - description = "Specifies the address prefix of the AKS subnet" - default = ["10.0.0.0/8"] - type = list(string) -} - variable "system_node_pool_subnet_name" { description = "Specifies the name of the subnet that hosts the system node pool" default = "SystemSubnet" type = string } -variable "system_node_pool_subnet_address_prefix" { - description = "Specifies the address prefix of the subnet that hosts the system node pool" - default = ["10.240.0.0/16"] - type = list(string) -} - variable "user_node_pool_subnet_name" { description = "Specifies the name of the subnet that hosts the user node pool" default = "UserSubnet" type = string } -variable "user_node_pool_subnet_address_prefix" { - description = "Specifies the address prefix of the subnet that hosts the user node pool" - type = list(string) - default = ["10.241.0.0/16"] -} - variable "pod_subnet_name" { description = "Specifies the name of the jumpbox subnet" default = "PodSubnet" type = string } -variable "pod_subnet_address_prefix" { - description = "Specifies the address prefix of the jumbox subnet" - default = ["10.242.0.0/16"] - type = list(string) -} - variable "vm_subnet_name" { description = "Specifies the name of the jumpbox subnet" default = "VmSubnet" type = string } -variable "vm_subnet_address_prefix" { - description = "Specifies the address prefix of the jumbox subnet" - default = ["10.243.1.0/24"] - type = list(string) -} - -variable "bastion_subnet_address_prefix" { - description = "Specifies the address prefix of the firewall subnet" - default = ["10.243.2.0/24"] - type = list(string) -} - variable "aks_cluster_name" { description = "(Required) Specifies the name of the AKS cluster." default = "Aks" type = string } -variable "private_cluster_enabled" { - description = "(Optional) Specifies wether the AKS cluster be private or not." - default = false - type = bool -} - -variable "role_based_access_control_enabled" { - description = "(Required) Is Role Based Access Control Enabled? Changing this forces a new resource to be created." - default = true - type = bool -} - -variable "admin_group_object_ids" { - description = "(Optional) A list of Object IDs of Azure Active Directory Groups which should have Admin Role on the Cluster." - default = [] - type = list(string) -} - -variable "azure_rbac_enabled" { - description = "(Optional) Is Role Based Access Control based on Azure AD enabled?" - default = true - type = bool -} - -variable "sku_tier" { - description = "(Optional) The SKU Tier that should be used for this Kubernetes Cluster. Possible values are Free and Paid (which includes the Uptime SLA). Defaults to Free." - default = "Free" - type = string - - validation { - condition = contains( ["Free", "Paid"], var.sku_tier) - error_message = "The sku tier is invalid." - } -} - variable "kubernetes_version" { description = "Specifies the AKS Kubernetes version" default = "1.29.10" @@ -194,24 +104,6 @@ variable "system_node_pool_vm_size" { type = string } -variable "system_node_pool_availability_zones" { - description = "Specifies the availability zones of the system node pool" - default = ["1", "2", "3"] - type = list(string) -} - -variable "network_dns_service_ip" { - description = "Specifies the DNS service IP" - default = "10.2.0.10" - type = string -} - -variable "network_service_cidr" { - description = "Specifies the service CIDR" - default = "10.2.0.0/24" - type = string -} - variable "network_plugin" { description = "Specifies the network plugin of the AKS cluster" default = "azure" @@ -266,42 +158,6 @@ variable "user_node_pool_availability_zones" { default = ["1", "2", "3"] } -variable "user_node_pool_enable_host_encryption" { - description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." - type = bool - default = false -} - -variable "user_node_pool_enable_node_public_ip" { - description = "(Optional) Should each node have a Public IP Address? Defaults to false. Changing this forces a new resource to be created." - type = bool - default = false -} - -variable "user_node_pool_max_pods" { - description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." - type = number - default = 50 -} - -variable "user_node_pool_mode" { - description = "(Optional) Should this Node Pool be used for System or User resources? Possible values are System and User. Defaults to User." - type = string - default = "User" -} - -variable "user_node_pool_node_labels" { - description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." - type = map(any) - default = {} -} - -variable "user_node_pool_node_taints" { - description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." - type = list(string) - default = [] -} - variable "user_node_pool_os_disk_type" { description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." type = string @@ -331,93 +187,6 @@ variable "storage_account_kind" { } } -variable "storage_account_tier" { - description = "(Optional) Specifies the account tier of the storage account" - default = "Standard" - type = string - - validation { - condition = contains(["Standard", "Premium"], var.storage_account_tier) - error_message = "The account tier of the storage account is invalid." - } -} - -variable "acr_name" { - description = "Specifies the name of the container registry" - type = string - default = "Acr" -} - -variable "acr_sku" { - description = "Specifies the name of the container registry" - type = string - default = "Premium" - - validation { - condition = contains(["Basic", "Standard", "Premium"], var.acr_sku) - error_message = "The container registry sku is invalid." - } -} - -variable "acr_admin_enabled" { - description = "Specifies whether admin is enabled for the container registry" - type = bool - default = true -} - -variable "acr_georeplication_locations" { - description = "(Optional) A list of Azure locations where the container registry should be geo-replicated." - type = list(string) - default = [] -} - -variable "tags" { - description = "(Optional) Specifies tags for all the resources" - default = { - createdWith = "Terraform" - } -} - -variable "bastion_host_name" { - description = "(Optional) Specifies the name of the bastion host" - default = "BastionHost" - type = string -} - -variable "storage_account_replication_type" { - description = "(Optional) Specifies the replication type of the storage account" - default = "LRS" - type = string - - validation { - condition = contains(["LRS", "ZRS", "GRS", "GZRS", "RA-GRS", "RA-GZRS"], var.storage_account_replication_type) - error_message = "The replication type of the storage account is invalid." - } -} - -variable "key_vault_name" { - description = "Specifies the name of the key vault." - type = string - default = "KeyVault" -} - -variable "key_vault_sku_name" { - description = "(Required) The Name of the SKU used for this Key Vault. Possible values are standard and premium." - type = string - default = "standard" - - validation { - condition = contains(["standard", "premium" ], var.key_vault_sku_name) - error_message = "The sku name of the key vault is invalid." - } -} - -variable"key_vault_enabled_for_deployment" { - description = "(Optional) Boolean flag to specify whether Azure Virtual Machines are permitted to retrieve certificates stored as secrets from the key vault. Defaults to false." - type = bool - default = true -} - variable"key_vault_enabled_for_disk_encryption" { description = " (Optional) Boolean flag to specify whether Azure Disk Encryption is permitted to retrieve secrets from the vault and unwrap keys. Defaults to false." type = bool @@ -637,24 +406,6 @@ variable "deployment_script_name" { default = "BashScript" } -variable "deployment_script_azure_cli_version" { - description = "(Required) Azure CLI module version to be used." - type = string - default = "2.9.1" -} - -variable "deployment_script_managed_identity_name" { - description = "Specifies the name of the user-defined managed identity used by the deployment script." - type = string - default = "ScriptManagedIdentity" -} - -variable "deployment_script_primary_script_uri" { - description = "(Optional) Uri for the script. This is the entry point for the external script. Changing this forces a new Resource Deployment Script to be created." - type = string - default = "https://paolosalvatori.blob.core.windows.net/scripts/install-nginx-via-helm-and-create-sa.sh" -} - resource "azurerm_resource_group" "rg" { name = var.name_prefix == null ? "${random_string.prefix.result}${var.resource_group_name}" : "${var.name_prefix}${var.resource_group_name}" location = var.location @@ -665,35 +416,40 @@ module "log_analytics_workspace" { name = var.name_prefix == null ? "${random_string.prefix.result}${var.log_analytics_workspace_name}" : "${var.name_prefix}${var.log_analytics_workspace_name}" location = var.location resource_group_name = azurerm_resource_group.rg.name - solution_plan_map = var.solution_plan_map + solution_plan_map = { + ContainerInsights= { + product = "OMSGallery/ContainerInsights" + publisher = "Microsoft" + } + } } module "virtual_network" { source = "./modules/virtual_network" resource_group_name = azurerm_resource_group.rg.name location = var.location - vnet_name = var.name_prefix == null ? "${random_string.prefix.result}${var.vnet_name}" : "${var.name_prefix}${var.vnet_name}" - address_space = var.vnet_address_space + vnet_name = "AksVNet" + address_space = ["10.0.0.0/8"] log_analytics_workspace_id = module.log_analytics_workspace.id subnets = [ { name : var.system_node_pool_subnet_name - address_prefixes : var.system_node_pool_subnet_address_prefix + address_prefixes : ["10.240.0.0/16"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false delegation: null }, { name : var.user_node_pool_subnet_name - address_prefixes : var.user_node_pool_subnet_address_prefix + address_prefixes : ["10.241.0.0/16"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false delegation: null }, { name : var.pod_subnet_name - address_prefixes : var.pod_subnet_address_prefix + address_prefixes : ["10.242.0.0/16"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false delegation = { @@ -706,14 +462,14 @@ module "virtual_network" { }, { name : var.vm_subnet_name - address_prefixes : var.vm_subnet_address_prefix + address_prefixes : ["10.243.1.0/24"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false delegation: null }, { name : "AzureBastionSubnet" - address_prefixes : var.bastion_subnet_address_prefix + address_prefixes : ["10.243.2.0/24"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false delegation: null @@ -734,12 +490,11 @@ module "nat_gateway" { module "container_registry" { source = "./modules/container_registry" - name = var.name_prefix == null ? "${random_string.prefix.result}${var.acr_name}" : "${var.name_prefix}${var.acr_name}" + name = "${var.name_prefix}Acr" resource_group_name = azurerm_resource_group.rg.name location = var.location - sku = var.acr_sku - admin_enabled = var.acr_admin_enabled - georeplication_locations = var.acr_georeplication_locations + sku = "Basic" + admin_enabled = true log_analytics_workspace_id = module.log_analytics_workspace.id } @@ -751,25 +506,24 @@ module "aks_cluster" { resource_group_id = azurerm_resource_group.rg.id kubernetes_version = var.kubernetes_version dns_prefix = lower(var.aks_cluster_name) - private_cluster_enabled = var.private_cluster_enabled - sku_tier = var.sku_tier + private_cluster_enabled = false + sku_tier = "Free" system_node_pool_name = var.system_node_pool_name system_node_pool_vm_size = var.system_node_pool_vm_size vnet_subnet_id = module.virtual_network.subnet_ids[var.system_node_pool_subnet_name] pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] - system_node_pool_availability_zones = var.system_node_pool_availability_zones + system_node_pool_availability_zones = ["1", "2", "3"] system_node_pool_node_labels = var.system_node_pool_node_labels system_node_pool_max_pods = var.system_node_pool_max_pods system_node_pool_os_disk_type = var.system_node_pool_os_disk_type - network_dns_service_ip = var.network_dns_service_ip + network_dns_service_ip = "10.2.0.10" network_plugin = var.network_plugin outbound_type = "userAssignedNATGateway" - network_service_cidr = var.network_service_cidr + network_service_cidr = "10.2.0.0/24" log_analytics_workspace_id = module.log_analytics_workspace.id - role_based_access_control_enabled = var.role_based_access_control_enabled + role_based_access_control_enabled = true tenant_id = data.azurerm_client_config.current.tenant_id - admin_group_object_ids = var.admin_group_object_ids - azure_rbac_enabled = var.azure_rbac_enabled + azure_rbac_enabled = true admin_username = var.admin_username keda_enabled = var.keda_enabled vertical_pod_autoscaler_enabled = var.vertical_pod_autoscaler_enabled @@ -792,16 +546,14 @@ module "node_pool" { kubernetes_cluster_id = module.aks_cluster.id name = var.user_node_pool_name vm_size = var.user_node_pool_vm_size - mode = var.user_node_pool_mode - node_labels = var.user_node_pool_node_labels - node_taints = var.user_node_pool_node_taints + mode = "User" availability_zones = var.user_node_pool_availability_zones vnet_subnet_id = module.virtual_network.subnet_ids[var.user_node_pool_subnet_name] pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] - enable_host_encryption = var.user_node_pool_enable_host_encryption - enable_node_public_ip = var.user_node_pool_enable_node_public_ip + enable_host_encryption = false + enable_node_public_ip = false orchestrator_version = var.kubernetes_version - max_pods = var.user_node_pool_max_pods + max_pods = 50 os_type = var.user_node_pool_os_type priority = var.user_node_pool_priority } @@ -823,12 +575,6 @@ resource "azurerm_user_assigned_identity" "aks_workload_identity" { name = var.name_prefix == null ? "${random_string.prefix.result}${var.workload_managed_identity_name}" : "${var.name_prefix}${var.workload_managed_identity_name}" resource_group_name = azurerm_resource_group.rg.name location = var.location - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_role_assignment" "cognitive_services_user_assignment" { @@ -867,30 +613,29 @@ module "storage_account" { location = var.location resource_group_name = azurerm_resource_group.rg.name account_kind = var.storage_account_kind - account_tier = var.storage_account_tier - replication_type = var.storage_account_replication_type + account_tier = "Standard" + replication_type = "LRS" } module "bastion_host" { source = "./modules/bastion_host" - name = var.name_prefix == null ? "${random_string.prefix.result}${var.bastion_host_name}" : "${var.name_prefix}${var.bastion_host_name}" + name = "${var.name_prefix}BastionHost" location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids["AzureBastionSubnet"] log_analytics_workspace_id = module.log_analytics_workspace.id log_analytics_retention_days = var.log_analytics_retention_days - tags = var.tags } module "key_vault" { source = "./modules/key_vault" - name = var.name_prefix == null ? "${random_string.prefix.result}${var.key_vault_name}" : "${var.name_prefix}${var.key_vault_name}" + name = "${var.name_prefix}KeyVault" location = var.location resource_group_name = azurerm_resource_group.rg.name tenant_id = data.azurerm_client_config.current.tenant_id - sku_name = var.key_vault_sku_name - enabled_for_deployment = var.key_vault_enabled_for_deployment - enabled_for_disk_encryption = var.key_vault_enabled_for_disk_encryption + sku_name = "standard" + enabled_for_deployment = true + enabled_for_disk_encryption = true enabled_for_template_deployment = var.key_vault_enabled_for_template_deployment enable_rbac_authorization = var.key_vault_enable_rbac_authorization purge_protection_enabled = var.key_vault_purge_protection_enabled @@ -1006,14 +751,14 @@ module "deployment_script" { name = var.name_prefix == null ? "${random_string.prefix.result}${var.deployment_script_name}" : "${var.name_prefix}${var.deployment_script_name}" location = var.location resource_group_name = azurerm_resource_group.rg.name - azure_cli_version = var.deployment_script_azure_cli_version - managed_identity_name = var.name_prefix == null ? "${random_string.prefix.result}${var.deployment_script_managed_identity_name}" : "${var.name_prefix}${var.deployment_script_managed_identity_name}" + azure_cli_version = "2.9.1" + managed_identity_name = "${var.name_prefix}ScriptManagedIdentity" aks_cluster_name = module.aks_cluster.name hostname = "${var.subdomain}.${var.domain}" namespace = var.namespace service_account_name = var.service_account_name email = var.email - primary_script_uri = var.deployment_script_primary_script_uri + primary_script_uri = "https://paolosalvatori.blob.core.windows.net/scripts/install-nginx-via-helm-and-create-sa.sh" tenant_id = data.azurerm_client_config.current.tenant_id subscription_id = data.azurerm_client_config.current.subscription_id workload_managed_identity_client_id = azurerm_user_assigned_identity.aks_workload_identity.client_id From 0f0b3b3ff4a30177d7a6ce54130d399a7f66d624 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 13:50:11 -0800 Subject: [PATCH 024/119] More inlining --- .../AksOpenAiTerraform/terraform/main.tf | 370 +++--------------- 1 file changed, 60 insertions(+), 310 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 971960f18..3434e40c5 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -11,10 +11,6 @@ provider "azurerm" { features {} } -locals { - storage_account_prefix = "boot" -} - data "azurerm_client_config" "current" { } @@ -104,12 +100,6 @@ variable "system_node_pool_vm_size" { type = string } -variable "network_plugin" { - description = "Specifies the network plugin of the AKS cluster" - default = "azure" - type = string -} - variable "system_node_pool_name" { description = "Specifies the name of the system node pool" default = "system" @@ -122,24 +112,6 @@ variable "system_node_pool_max_pods" { default = 50 } -variable "system_node_pool_node_labels" { - description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." - type = map(any) - default = {} -} - -variable "system_node_pool_node_taints" { - description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." - type = list(string) - default = ["CriticalAddonsOnly=true:NoSchedule"] -} - -variable "system_node_pool_os_disk_type" { - description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." - type = string - default = "Ephemeral" -} - variable "user_node_pool_name" { description = "(Required) Specifies the name of the node pool." type = string @@ -152,236 +124,6 @@ variable "user_node_pool_vm_size" { default = "Standard_D8ds_v5" } -variable "user_node_pool_availability_zones" { - description = "(Optional) A list of Availability Zones where the Nodes in this Node Pool should be created in. Changing this forces a new resource to be created." - type = list(string) - default = ["1", "2", "3"] -} - -variable "user_node_pool_os_disk_type" { - description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." - type = string - default = "Ephemeral" -} - -variable "user_node_pool_os_type" { - description = "(Optional) The Operating System which should be used for this Node Pool. Changing this forces a new resource to be created. Possible values are Linux and Windows. Defaults to Linux." - type = string - default = "Linux" -} - -variable "user_node_pool_priority" { - description = "(Optional) The Priority for Virtual Machines within the Virtual Machine Scale Set that powers this Node Pool. Possible values are Regular and Spot. Defaults to Regular. Changing this forces a new resource to be created." - type = string - default = "Regular" -} - -variable "storage_account_kind" { - description = "(Optional) Specifies the account kind of the storage account" - default = "StorageV2" - type = string - - validation { - condition = contains(["Storage", "StorageV2"], var.storage_account_kind) - error_message = "The account kind of the storage account is invalid." - } -} - -variable"key_vault_enabled_for_disk_encryption" { - description = " (Optional) Boolean flag to specify whether Azure Disk Encryption is permitted to retrieve secrets from the vault and unwrap keys. Defaults to false." - type = bool - default = true -} - -variable"key_vault_enabled_for_template_deployment" { - description = "(Optional) Boolean flag to specify whether Azure Resource Manager is permitted to retrieve secrets from the key vault. Defaults to false." - type = bool - default = true -} - -variable"key_vault_enable_rbac_authorization" { - description = "(Optional) Boolean flag to specify whether Azure Key Vault uses Role Based Access Control (RBAC) for authorization of data actions. Defaults to false." - type = bool - default = true -} - -variable"key_vault_purge_protection_enabled" { - description = "(Optional) Is Purge Protection enabled for this Key Vault? Defaults to false." - type = bool - default = false -} - -variable "key_vault_soft_delete_retention_days" { - description = "(Optional) The number of days that items should be retained for once soft-deleted. This value can be between 7 and 90 (the default) days." - type = number - default = 30 -} - -variable "key_vault_bypass" { - description = "(Required) Specifies which traffic can bypass the network rules. Possible values are AzureServices and None." - type = string - default = "AzureServices" - - validation { - condition = contains(["AzureServices", "None" ], var.key_vault_bypass) - error_message = "The valut of the bypass property of the key vault is invalid." - } -} - -variable "key_vault_default_action" { - description = "(Required) The Default Action to use when no rules match from ip_rules / virtual_network_subnet_ids. Possible values are Allow and Deny." - type = string - default = "Allow" - - validation { - condition = contains(["Allow", "Deny" ], var.key_vault_default_action) - error_message = "The value of the default action property of the key vault is invalid." - } -} - -variable "admin_username" { - description = "(Required) Specifies the admin username of the jumpbox virtual machine and AKS worker nodes." - type = string - default = "azadmin" -} - -variable "keda_enabled" { - description = "(Optional) Specifies whether KEDA Autoscaler can be used for workloads." - type = bool - default = true -} - -variable "vertical_pod_autoscaler_enabled" { - description = "(Optional) Specifies whether Vertical Pod Autoscaler should be enabled." - type = bool - default = true -} - -variable "workload_identity_enabled" { - description = "(Optional) Specifies whether Azure AD Workload Identity should be enabled for the Cluster. Defaults to false." - type = bool - default = true -} - -variable "oidc_issuer_enabled" { - description = "(Optional) Enable or Disable the OIDC issuer URL." - type = bool - default = true -} - -variable "open_service_mesh_enabled" { - description = "(Optional) Is Open Service Mesh enabled? For more details, please visit Open Service Mesh for AKS." - type = bool - default = true -} - -variable "image_cleaner_enabled" { - description = "(Optional) Specifies whether Image Cleaner is enabled." - type = bool - default = true -} - -variable "azure_policy_enabled" { - description = "(Optional) Should the Azure Policy Add-On be enabled? For more details please visit Understand Azure Policy for Azure Kubernetes Service" - type = bool - default = true -} - -variable "http_application_routing_enabled" { - description = "(Optional) Should HTTP Application Routing be enabled?" - type = bool - default = false -} - -variable "openai_name" { - description = "(Required) Specifies the name of the Azure OpenAI Service" - type = string - default = "OpenAi" -} - -variable "openai_sku_name" { - description = "(Optional) Specifies the sku name for the Azure OpenAI Service" - type = string - default = "S0" -} - -variable "openai_custom_subdomain_name" { - description = "(Optional) Specifies the custom subdomain name of the Azure OpenAI Service" - type = string - nullable = true - default = "" -} - -variable "openai_public_network_access_enabled" { - description = "(Optional) Specifies whether public network access is allowed for the Azure OpenAI Service" - type = bool - default = true -} - -variable "openai_deployments" { - description = "(Optional) Specifies the deployments of the Azure OpenAI Service" - type = list(object({ - name = string - model = object({ - name = string - version = string - }) - rai_policy_name = string - })) - default = [ - { - name = "gpt-35-turbo" - model = { - name = "gpt-35-turbo" - version = "0301" - } - rai_policy_name = "" - } - ] -} - -variable "nat_gateway_name" { - description = "(Required) Specifies the name of the Azure OpenAI Service" - type = string - default = "NatGateway" -} - -variable "nat_gateway_sku_name" { - description = "(Optional) The SKU which should be used. At this time the only supported value is Standard. Defaults to Standard" - type = string - default = "Standard" -} - -variable "nat_gateway_idle_timeout_in_minutes" { - description = "(Optional) The idle timeout which should be used in minutes. Defaults to 4." - type = number - default = 4 -} - -variable "nat_gateway_zones" { - description = " (Optional) A list of Availability Zones in which this NAT Gateway should be located. Changing this forces a new NAT Gateway to be created." - type = list(string) - default = ["1"] -} - -variable "workload_managed_identity_name" { - description = "(Required) Specifies the name of the workload user-defined managed identity." - type = string - default = "WorkloadManagedIdentity" -} - -variable "subdomain" { - description = "Specifies the subdomain of the Kubernetes ingress object." - type = string - default = "magic8ball" -} - -variable "domain" { - description = "Specifies the domain of the Kubernetes ingress object." - type = string - default = "contoso.com" -} - variable "namespace" { description = "Specifies the namespace of the workload application that accesses the Azure OpenAI Service." type = string @@ -400,22 +142,17 @@ variable "email" { default = "paolos@microsoft.com" } -variable "deployment_script_name" { - description = "(Required) Specifies the name of the Azure OpenAI Service" - type = string - default = "BashScript" -} - resource "azurerm_resource_group" "rg" { - name = var.name_prefix == null ? "${random_string.prefix.result}${var.resource_group_name}" : "${var.name_prefix}${var.resource_group_name}" + name = "${var.name_prefix}${var.resource_group_name}" location = var.location } module "log_analytics_workspace" { source = "./modules/log_analytics" - name = var.name_prefix == null ? "${random_string.prefix.result}${var.log_analytics_workspace_name}" : "${var.name_prefix}${var.log_analytics_workspace_name}" + name = "${var.name_prefix}${var.log_analytics_workspace_name}" location = var.location resource_group_name = azurerm_resource_group.rg.name + solution_plan_map = { ContainerInsights= { product = "OMSGallery/ContainerInsights" @@ -426,12 +163,13 @@ module "log_analytics_workspace" { module "virtual_network" { source = "./modules/virtual_network" - resource_group_name = azurerm_resource_group.rg.name - location = var.location vnet_name = "AksVNet" - address_space = ["10.0.0.0/8"] + location = var.location + resource_group_name = azurerm_resource_group.rg.name + log_analytics_workspace_id = module.log_analytics_workspace.id - + + address_space = ["10.0.0.0/8"] subnets = [ { name : var.system_node_pool_subnet_name @@ -479,31 +217,35 @@ module "virtual_network" { module "nat_gateway" { source = "./modules/nat_gateway" - name = var.name_prefix == null ? "${random_string.prefix.result}${var.nat_gateway_name}" : "${var.name_prefix}${var.nat_gateway_name}" - resource_group_name = azurerm_resource_group.rg.name + name = "${var.name_prefix}NatGateway" location = var.location - sku_name = var.nat_gateway_sku_name - idle_timeout_in_minutes = var.nat_gateway_idle_timeout_in_minutes - zones = var.nat_gateway_zones + resource_group_name = azurerm_resource_group.rg.name + + sku_name = "Standard" + idle_timeout_in_minutes = 4 + zones = ["1"] subnet_ids = module.virtual_network.subnet_ids } module "container_registry" { source = "./modules/container_registry" name = "${var.name_prefix}Acr" - resource_group_name = azurerm_resource_group.rg.name location = var.location + resource_group_name = azurerm_resource_group.rg.name + + log_analytics_workspace_id = module.log_analytics_workspace.id + sku = "Basic" admin_enabled = true - log_analytics_workspace_id = module.log_analytics_workspace.id } module "aks_cluster" { source = "./modules/aks" - name = var.name_prefix == null ? "${random_string.prefix.result}${var.aks_cluster_name}" : "${var.name_prefix}${var.aks_cluster_name}" + name = "${var.name_prefix}${var.aks_cluster_name}" location = var.location resource_group_name = azurerm_resource_group.rg.name resource_group_id = azurerm_resource_group.rg.id + kubernetes_version = var.kubernetes_version dns_prefix = lower(var.aks_cluster_name) private_cluster_enabled = false @@ -513,26 +255,25 @@ module "aks_cluster" { vnet_subnet_id = module.virtual_network.subnet_ids[var.system_node_pool_subnet_name] pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] system_node_pool_availability_zones = ["1", "2", "3"] - system_node_pool_node_labels = var.system_node_pool_node_labels system_node_pool_max_pods = var.system_node_pool_max_pods - system_node_pool_os_disk_type = var.system_node_pool_os_disk_type + system_node_pool_os_disk_type = "Ephemeral" network_dns_service_ip = "10.2.0.10" - network_plugin = var.network_plugin + network_plugin = "azure" outbound_type = "userAssignedNATGateway" network_service_cidr = "10.2.0.0/24" log_analytics_workspace_id = module.log_analytics_workspace.id role_based_access_control_enabled = true tenant_id = data.azurerm_client_config.current.tenant_id azure_rbac_enabled = true - admin_username = var.admin_username - keda_enabled = var.keda_enabled - vertical_pod_autoscaler_enabled = var.vertical_pod_autoscaler_enabled - workload_identity_enabled = var.workload_identity_enabled - oidc_issuer_enabled = var.oidc_issuer_enabled - open_service_mesh_enabled = var.open_service_mesh_enabled - image_cleaner_enabled = var.image_cleaner_enabled - azure_policy_enabled = var.azure_policy_enabled - http_application_routing_enabled = var.http_application_routing_enabled + admin_username = "${var.name_prefix}-azadmin" + keda_enabled = true + vertical_pod_autoscaler_enabled = true + workload_identity_enabled = true + oidc_issuer_enabled = true + open_service_mesh_enabled = true + image_cleaner_enabled = true + azure_policy_enabled = true + http_application_routing_enabled = false depends_on = [ module.nat_gateway, @@ -542,37 +283,46 @@ module "aks_cluster" { module "node_pool" { source = "./modules/node_pool" + name = var.user_node_pool_name resource_group_name = azurerm_resource_group.rg.name kubernetes_cluster_id = module.aks_cluster.id - name = var.user_node_pool_name vm_size = var.user_node_pool_vm_size mode = "User" - availability_zones = var.user_node_pool_availability_zones + availability_zones = ["1", "2", "3"] vnet_subnet_id = module.virtual_network.subnet_ids[var.user_node_pool_subnet_name] pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] enable_host_encryption = false enable_node_public_ip = false orchestrator_version = var.kubernetes_version max_pods = 50 - os_type = var.user_node_pool_os_type - priority = var.user_node_pool_priority + os_type = "Linux" + priority = "Regular" } module "openai" { source = "./modules/openai" - name = var.name_prefix == null ? "${random_string.prefix.result}${var.openai_name}" : "${var.name_prefix}${var.openai_name}" + name = "${var.name_prefix}OpenAi" location = var.location resource_group_name = azurerm_resource_group.rg.name - sku_name = var.openai_sku_name - deployments = var.openai_deployments - custom_subdomain_name = var.openai_custom_subdomain_name == "" || var.openai_custom_subdomain_name == null ? var.name_prefix == null ? lower("${random_string.prefix.result}${var.openai_name}") : lower("${var.name_prefix}${var.openai_name}") : lower(var.openai_custom_subdomain_name) - public_network_access_enabled = var.openai_public_network_access_enabled + sku_name = "S0" + deployments = [ + { + name = "gpt-35-turbo" + model = { + name = "gpt-35-turbo" + version = "0301" + } + rai_policy_name = "" + } + ] + custom_subdomain_name = lower("${var.name_prefix}OpenAi") + public_network_access_enabled = true log_analytics_workspace_id = module.log_analytics_workspace.id log_analytics_retention_days = var.log_analytics_retention_days } resource "azurerm_user_assigned_identity" "aks_workload_identity" { - name = var.name_prefix == null ? "${random_string.prefix.result}${var.workload_managed_identity_name}" : "${var.name_prefix}${var.workload_managed_identity_name}" + name = "${var.name_prefix}WorkloadManagedIdentity" resource_group_name = azurerm_resource_group.rg.name location = var.location } @@ -609,10 +359,10 @@ resource "azurerm_role_assignment" "acr_pull_assignment" { module "storage_account" { source = "./modules/storage_account" - name = "${local.storage_account_prefix}${random_string.storage_account_suffix.result}" + name = "boot${random_string.storage_account_suffix.result}" location = var.location resource_group_name = azurerm_resource_group.rg.name - account_kind = var.storage_account_kind + account_kind = "StorageV2" account_tier = "Standard" replication_type = "LRS" } @@ -636,12 +386,12 @@ module "key_vault" { sku_name = "standard" enabled_for_deployment = true enabled_for_disk_encryption = true - enabled_for_template_deployment = var.key_vault_enabled_for_template_deployment - enable_rbac_authorization = var.key_vault_enable_rbac_authorization - purge_protection_enabled = var.key_vault_purge_protection_enabled - soft_delete_retention_days = var.key_vault_soft_delete_retention_days - bypass = var.key_vault_bypass - default_action = var.key_vault_default_action + enabled_for_template_deployment = true + enable_rbac_authorization = true + purge_protection_enabled = false + soft_delete_retention_days = 30 + bypass = "AzureServices" + default_action = "Allow" log_analytics_workspace_id = module.log_analytics_workspace.id log_analytics_retention_days = var.log_analytics_retention_days } @@ -748,13 +498,13 @@ module "blob_private_endpoint" { module "deployment_script" { source = "./modules/deployment_script" - name = var.name_prefix == null ? "${random_string.prefix.result}${var.deployment_script_name}" : "${var.name_prefix}${var.deployment_script_name}" + name = "${var.name_prefix}BashScript" location = var.location resource_group_name = azurerm_resource_group.rg.name azure_cli_version = "2.9.1" managed_identity_name = "${var.name_prefix}ScriptManagedIdentity" aks_cluster_name = module.aks_cluster.name - hostname = "${var.subdomain}.${var.domain}" + hostname = "magic8ball.contoso.com" namespace = var.namespace service_account_name = var.service_account_name email = var.email From f9c3b62dbace6735d8cb902d4d1bbb046fde96ad Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 14:38:35 -0800 Subject: [PATCH 025/119] Massive clean up WIP --- .../AksOpenAiTerraform/terraform/main.tf | 98 +------- .../terraform/modules/aks/main.tf | 105 +++----- .../terraform/modules/aks/outputs.tf | 39 --- .../terraform/modules/aks/ssh.tf | 5 - .../terraform/modules/aks/variables.tf | 237 +----------------- .../terraform/modules/bastion_host/main.tf | 13 - .../modules/container_registry/main.tf | 16 +- .../terraform/modules/firewall/main.tf | 22 -- .../modules/network_security_group/main.tf | 6 - 9 files changed, 47 insertions(+), 494 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 3434e40c5..18f0d9748 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -30,100 +30,49 @@ resource "random_string" "storage_account_suffix" { } variable "name_prefix" { - description = "A prefix for the name of all the resource groups and resources." type = string } variable "log_analytics_workspace_name" { - description = "Specifies the name of the log analytics workspace" default = "Workspace" type = string } variable "log_analytics_retention_days" { - description = "Specifies the number of days of the retention policy" type = number default = 30 } variable "location" { - description = "Specifies the location for the resource group and all the resources" default = "westus2" type = string } variable "resource_group_name" { - description = "Specifies the resource group name" default = "RG" type = string } variable "system_node_pool_subnet_name" { - description = "Specifies the name of the subnet that hosts the system node pool" default = "SystemSubnet" type = string } variable "user_node_pool_subnet_name" { - description = "Specifies the name of the subnet that hosts the user node pool" default = "UserSubnet" type = string } variable "pod_subnet_name" { - description = "Specifies the name of the jumpbox subnet" default = "PodSubnet" type = string } variable "vm_subnet_name" { - description = "Specifies the name of the jumpbox subnet" default = "VmSubnet" type = string } -variable "aks_cluster_name" { - description = "(Required) Specifies the name of the AKS cluster." - default = "Aks" - type = string -} - -variable "kubernetes_version" { - description = "Specifies the AKS Kubernetes version" - default = "1.29.10" - type = string -} - -variable "system_node_pool_vm_size" { - description = "Specifies the vm size of the system node pool" - default = "Standard_D8ds_v5" - type = string -} - -variable "system_node_pool_name" { - description = "Specifies the name of the system node pool" - default = "system" - type = string -} - -variable "system_node_pool_max_pods" { - description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." - type = number - default = 50 -} - -variable "user_node_pool_name" { - description = "(Required) Specifies the name of the node pool." - type = string - default = "user" -} - -variable "user_node_pool_vm_size" { - description = "(Required) The SKU which should be used for the Virtual Machines used in this Node Pool. Changing this forces a new resource to be created." - type = string - default = "Standard_D8ds_v5" -} - variable "namespace" { description = "Specifies the namespace of the workload application that accesses the Azure OpenAI Service." type = string @@ -245,60 +194,15 @@ module "aks_cluster" { location = var.location resource_group_name = azurerm_resource_group.rg.name resource_group_id = azurerm_resource_group.rg.id - kubernetes_version = var.kubernetes_version - dns_prefix = lower(var.aks_cluster_name) - private_cluster_enabled = false sku_tier = "Free" - system_node_pool_name = var.system_node_pool_name - system_node_pool_vm_size = var.system_node_pool_vm_size - vnet_subnet_id = module.virtual_network.subnet_ids[var.system_node_pool_subnet_name] - pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] - system_node_pool_availability_zones = ["1", "2", "3"] - system_node_pool_max_pods = var.system_node_pool_max_pods - system_node_pool_os_disk_type = "Ephemeral" - network_dns_service_ip = "10.2.0.10" - network_plugin = "azure" - outbound_type = "userAssignedNATGateway" - network_service_cidr = "10.2.0.0/24" - log_analytics_workspace_id = module.log_analytics_workspace.id - role_based_access_control_enabled = true - tenant_id = data.azurerm_client_config.current.tenant_id - azure_rbac_enabled = true - admin_username = "${var.name_prefix}-azadmin" - keda_enabled = true - vertical_pod_autoscaler_enabled = true - workload_identity_enabled = true - oidc_issuer_enabled = true - open_service_mesh_enabled = true - image_cleaner_enabled = true - azure_policy_enabled = true - http_application_routing_enabled = false - + depends_on = [ module.nat_gateway, module.container_registry ] } -module "node_pool" { - source = "./modules/node_pool" - name = var.user_node_pool_name - resource_group_name = azurerm_resource_group.rg.name - kubernetes_cluster_id = module.aks_cluster.id - vm_size = var.user_node_pool_vm_size - mode = "User" - availability_zones = ["1", "2", "3"] - vnet_subnet_id = module.virtual_network.subnet_ids[var.user_node_pool_subnet_name] - pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] - enable_host_encryption = false - enable_node_public_ip = false - orchestrator_version = var.kubernetes_version - max_pods = 50 - os_type = "Linux" - priority = "Regular" -} - module "openai" { source = "./modules/openai" name = "${var.name_prefix}OpenAi" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index 6178c43f3..eb331b3d0 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -1,24 +1,7 @@ -terraform { - required_providers { - azapi = { - source = "Azure/azapi" - version = "~>2.0.1" - } - } -} - resource "azurerm_user_assigned_identity" "aks_identity" { + name = "${var.name}Identity" resource_group_name = var.resource_group_name location = var.location - tags = var.tags - - name = "${var.name}Identity" - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_kubernetes_cluster" "aks_cluster" { @@ -26,36 +9,27 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { location = var.location resource_group_name = var.resource_group_name kubernetes_version = var.kubernetes_version - dns_prefix = var.dns_prefix - private_cluster_enabled = var.private_cluster_enabled + dns_prefix = lower(var.name) + private_cluster_enabled = false automatic_upgrade_channel = "stable" sku_tier = var.sku_tier - workload_identity_enabled = var.workload_identity_enabled - oidc_issuer_enabled = var.oidc_issuer_enabled - open_service_mesh_enabled = var.open_service_mesh_enabled - image_cleaner_enabled = var.image_cleaner_enabled - azure_policy_enabled = var.azure_policy_enabled + workload_identity_enabled = true + oidc_issuer_enabled = true + open_service_mesh_enabled = true + image_cleaner_enabled = true image_cleaner_interval_hours = 72 - http_application_routing_enabled = var.http_application_routing_enabled + azure_policy_enabled = true + http_application_routing_enabled = false default_node_pool { - name = var.system_node_pool_name + name = "system" node_count = 1 vm_size = var.system_node_pool_vm_size - vnet_subnet_id = var.vnet_subnet_id - pod_subnet_id = var.pod_subnet_id - zones = var.system_node_pool_availability_zones - node_labels = var.system_node_pool_node_labels - max_pods = var.system_node_pool_max_pods - os_disk_type = var.system_node_pool_os_disk_type - tags = var.tags - } - - linux_profile { - admin_username = var.admin_username - ssh_key { - key_data = azapi_resource_action.ssh_public_key_gen.output.publicKey - } + vnet_subnet_id = module.virtual_network.subnet_ids[var.system_node_pool_subnet_name] + pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] + zones = ["1", "2", "3"] + max_pods = 50 + os_disk_type = "Ephemeral" } identity { @@ -64,44 +38,41 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { } network_profile { - dns_service_ip = var.network_dns_service_ip - network_plugin = var.network_plugin - outbound_type = var.outbound_type - service_cidr = var.network_service_cidr + dns_service_ip = "10.2.0.10" + network_plugin = "azure" + outbound_type = "userAssignedNATGateway" + service_cidr = "10.2.0.0/24" } oms_agent { msi_auth_for_monitoring_enabled = true - log_analytics_workspace_id = coalesce(var.oms_agent.log_analytics_workspace_id, var.log_analytics_workspace_id) - } - - dynamic "ingress_application_gateway" { - for_each = try(var.ingress_application_gateway.gateway_id, null) == null ? [] : [1] - - content { - gateway_id = var.ingress_application_gateway.gateway_id - subnet_cidr = var.ingress_application_gateway.subnet_cidr - subnet_id = var.ingress_application_gateway.subnet_id - } + log_analytics_workspace_id = var.log_analytics_workspace_id } azure_active_directory_role_based_access_control { - tenant_id = var.tenant_id - admin_group_object_ids = var.admin_group_object_ids - azure_rbac_enabled = var.azure_rbac_enabled + tenant_id = data.azurerm_client_config.current.tenant_id + azure_rbac_enabled = true } workload_autoscaler_profile { - keda_enabled = var.keda_enabled - vertical_pod_autoscaler_enabled = var.vertical_pod_autoscaler_enabled + keda_enabled = true + vertical_pod_autoscaler_enabled = true } +} - lifecycle { - ignore_changes = [ - kubernetes_version, - tags - ] - } +resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { + kubernetes_cluster_id = azurerm_kubernetes_cluster.aks_cluster.id + name = "user" + vm_size = var.user_node_pool_vm_size + mode = "User" + zones = ["1", "2", "3"] + vnet_subnet_id = module.virtual_network.subnet_ids[var.user_node_pool_subnet_name] + pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] + orchestrator_version = var.kubernetes_version + max_pods = 50 + os_disk_type = "Ephemeral" + os_type = "Linux" + priority = "Regular" } resource "azurerm_monitor_diagnostic_setting" "settings" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf deleted file mode 100644 index fd2e362d8..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf +++ /dev/null @@ -1,39 +0,0 @@ -output "name" { - value = azurerm_kubernetes_cluster.aks_cluster.name - description = "Specifies the name of the AKS cluster." -} - -output "id" { - value = azurerm_kubernetes_cluster.aks_cluster.id - description = "Specifies the resource id of the AKS cluster." -} - -output "aks_identity_principal_id" { - value = azurerm_user_assigned_identity.aks_identity.principal_id - description = "Specifies the principal id of the managed identity of the AKS cluster." -} - -output "kubelet_identity_object_id" { - value = azurerm_kubernetes_cluster.aks_cluster.kubelet_identity.0.object_id - description = "Specifies the object id of the kubelet identity of the AKS cluster." -} - -output "kube_config_raw" { - value = azurerm_kubernetes_cluster.aks_cluster.kube_config_raw - description = "Contains the Kubernetes config to be used by kubectl and other compatible tools." -} - -output "private_fqdn" { - value = azurerm_kubernetes_cluster.aks_cluster.private_fqdn - description = "The FQDN for the Kubernetes Cluster when private link has been enabled, which is only resolvable inside the Virtual Network used by the Kubernetes Cluster." -} - -output "node_resource_group" { - value = azurerm_kubernetes_cluster.aks_cluster.node_resource_group - description = "Specifies the resource id of the auto-generated Resource Group which contains the resources for this Managed Kubernetes Cluster." -} - -output "oidc_issuer_url" { - value = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url - description = "Specifies the URL of the OpenID Connect issuer used by this Kubernetes Cluster." -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf index 4cb7b3c37..364aa884e 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf @@ -1,6 +1,5 @@ resource "random_pet" "ssh_key_name" { prefix = "ssh" - separator = "" } resource "azapi_resource_action" "ssh_public_key_gen" { @@ -17,8 +16,4 @@ resource "azapi_resource" "ssh_public_key" { name = random_pet.ssh_key_name.id location = var.location parent_id = var.resource_group_id -} - -output "key_data" { - value = azapi_resource_action.ssh_public_key_gen.output.publicKey } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf index a054a87a0..54339d448 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf @@ -1,256 +1,33 @@ variable "name" { - description = "(Required) Specifies the name of the AKS cluster." type = string } variable "resource_group_name" { - description = "(Required) Specifies the name of the resource group." type = string } variable "resource_group_id" { - description = "(Required) Specifies the resource id of the resource group." type = string } variable "location" { - description = "(Required) Specifies the location where the AKS cluster will be deployed." type = string } -variable "dns_prefix" { - description = "(Optional) DNS prefix specified when creating the managed cluster. Changing this forces a new resource to be created." - type = string -} - -variable "private_cluster_enabled" { - description = "Should this Kubernetes Cluster have its API server only exposed on internal IP addresses? This provides a Private IP Address for the Kubernetes API on the Virtual Network where the Kubernetes Cluster is located. Defaults to false. Changing this forces a new resource to be created." - type = bool - default = false -} - -variable "azure_rbac_enabled" { - description = "(Optional) Is Role Based Access Control based on Azure AD enabled?" - default = true - type = bool -} - -variable "admin_group_object_ids" { - description = "(Optional) A list of Object IDs of Azure Active Directory Groups which should have Admin Role on the Cluster." - default = [] - type = list(string) -} - -variable "role_based_access_control_enabled" { - description = "(Required) Is Role Based Access Control Enabled? Changing this forces a new resource to be created." - default = true - type = bool -} - -variable "sku_tier" { - description = "(Optional) The SKU Tier that should be used for this Kubernetes Cluster. Possible values are Free and Paid (which includes the Uptime SLA). Defaults to Free." - default = "Free" - type = string - - validation { - condition = contains( ["Free", "Paid"], var.sku_tier) - error_message = "The sku tier is invalid." - } -} - variable "kubernetes_version" { - description = "Specifies the AKS Kubernetes version" - type = string -} - -variable "system_node_pool_vm_size" { - description = "Specifies the vm size of the system node pool" - type = string -} - -variable "system_node_pool_availability_zones" { - description = "Specifies the availability zones of the system node pool" - default = ["1", "2", "3"] - type = list(string) -} - -variable "network_dns_service_ip" { - description = "Specifies the DNS service IP" - default = "10.2.0.10" - type = string -} - -variable "network_service_cidr" { - description = "Specifies the service CIDR" - default = "10.2.0.0/24" - type = string -} - -variable "network_plugin" { - description = "Specifies the network plugin of the AKS cluster" - default = "azure" - type = string -} - -variable "outbound_type" { - description = "(Optional) The outbound (egress) routing method which should be used for this Kubernetes Cluster. Possible values are loadBalancer and userDefinedRouting. Defaults to loadBalancer." - type = string - default = "userDefinedRouting" - - validation { - condition = contains(["loadBalancer", "userDefinedRouting", "userAssignedNATGateway", "managedNATGateway"], var.outbound_type) - error_message = "The outbound type is invalid." - } -} - -variable "system_node_pool_name" { - description = "Specifies the name of the system node pool" - default = "system" - type = string -} - -variable "system_node_pool_subnet_name" { - description = "Specifies the name of the subnet that hosts the system node pool" - default = "SystemSubnet" - type = string -} - -variable "system_node_pool_subnet_address_prefix" { - description = "Specifies the address prefix of the subnet that hosts the system node pool" - default = ["10.0.0.0/20"] - type = list(string) -} - -variable "system_node_pool_max_pods" { - description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." - type = number - default = 50 + type = string } -variable "system_node_pool_node_labels" { - description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." - type = map(any) - default = {} -} - -variable "system_node_pool_os_disk_type" { - description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." - type = string - default = "Ephemeral" -} - -variable "log_analytics_workspace_id" { - description = "(Optional) The ID of the Log Analytics Workspace which the OMS Agent should send data to. Must be present if enabled is true." - type = string +variable sku_tier { + type = string } -variable "tenant_id" { - description = "(Required) The tenant id of the system assigned identity which is used by master components." +variable "system_node_pool_vm_size" { + default = "Standard_D8ds_v5" type = string } -variable "log_analytics_retention_days" { - description = "Specifies the number of days of the retention policy" - type = number - default = 30 -} - -variable "vnet_subnet_id" { - description = "(Optional) The ID of a Subnet where the Kubernetes Node Pool should exist. Changing this forces a new resource to be created." +variable "user_node_pool_vm_size" { + default = "Standard_D8ds_v5" type = string -} - -variable "pod_subnet_id" { - description = "(Optional) The ID of the Subnet where the pods in the system node pool should exist. Changing this forces a new resource to be created." - type = string - default = null -} - -variable "tags" { - description = "(Optional) Specifies the tags of the bastion host" - default = {} -} - -variable "oms_agent" { - description = "Specifies the OMS agent addon configuration." - type = object({ - enabled = bool - log_analytics_workspace_id = string - }) - default = { - enabled = true - log_analytics_workspace_id = null - } -} - -variable "ingress_application_gateway" { - description = "Specifies the Application Gateway Ingress Controller addon configuration." - type = object({ - enabled = bool - gateway_id = string - gateway_name = string - subnet_cidr = string - subnet_id = string - }) - default = { - enabled = false - gateway_id = null - gateway_name = null - subnet_cidr = null - subnet_id = null - } -} - -variable "admin_username" { - description = "(Required) Specifies the Admin Username for the AKS cluster worker nodes. Changing this forces a new resource to be created." - type = string - default = "azadmin" -} - -variable "keda_enabled" { - description = "(Optional) Specifies whether KEDA Autoscaler can be used for workloads." - type = bool - default = true -} - -variable "vertical_pod_autoscaler_enabled" { - description = "(Optional) Specifies whether Vertical Pod Autoscaler should be enabled." - type = bool - default = true -} - -variable "workload_identity_enabled" { - description = "(Optional) Specifies whether Azure AD Workload Identity should be enabled for the Cluster. Defaults to false." - type = bool - default = true -} - -variable "oidc_issuer_enabled" { - description = "(Optional) Enable or Disable the OIDC issuer URL." - type = bool - default = true -} - -variable "open_service_mesh_enabled" { - description = "(Optional) Is Open Service Mesh enabled? For more details, please visit Open Service Mesh for AKS." - type = bool - default = true -} - -variable "image_cleaner_enabled" { - description = "(Optional) Specifies whether Image Cleaner is enabled." - type = bool - default = true -} - -variable "azure_policy_enabled" { - description = "(Optional) Should the Azure Policy Add-On be enabled? For more details please visit Understand Azure Policy for Azure Kubernetes Service" - type = bool - default = true -} - -variable "http_application_routing_enabled" { - description = "(Optional) Should HTTP Application Routing be enabled?" - type = bool - default = false } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf index cbc9428cd..b7a3ac116 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf @@ -4,13 +4,6 @@ resource "azurerm_public_ip" "public_ip" { resource_group_name = var.resource_group_name allocation_method = "Static" sku = "Standard" - tags = var.tags - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_bastion_host" "bastion_host" { @@ -24,12 +17,6 @@ resource "azurerm_bastion_host" "bastion_host" { subnet_id = var.subnet_id public_ip_address_id = azurerm_public_ip.public_ip.id } - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_monitor_diagnostic_setting" "settings" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf index 44a9a669c..32f63c469 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf @@ -21,26 +21,12 @@ resource "azurerm_container_registry" "acr" { tags = var.tags } } - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_user_assigned_identity" "acr_identity" { + name = "${var.name}Identity" resource_group_name = var.resource_group_name location = var.location - tags = var.tags - - name = "${var.name}Identity" - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_monitor_diagnostic_setting" "settings" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf index 479cefb33..0d5474863 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf @@ -5,13 +5,6 @@ resource "azurerm_public_ip" "pip" { zones = var.zones allocation_method = "Static" sku = "Standard" - tags = var.tags - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_firewall" "firewall" { @@ -23,33 +16,18 @@ resource "azurerm_firewall" "firewall" { sku_name = var.sku_name sku_tier = var.sku_tier firewall_policy_id = azurerm_firewall_policy.policy.id - tags = var.tags - ip_configuration { name = "fw_ip_config" subnet_id = var.subnet_id public_ip_address_id = azurerm_public_ip.pip.id } - - lifecycle { - ignore_changes = [ - tags, - - ] - } } resource "azurerm_firewall_policy" "policy" { name = "${var.name}Policy" resource_group_name = var.resource_group_name location = var.location - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_firewall_policy_rule_collection_group" "policy" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf index b1a7589cb..ba9652eb2 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf @@ -24,12 +24,6 @@ resource "azurerm_network_security_group" "nsg" { destination_application_security_group_ids = try(security_rule.value.destination_application_security_group_ids, null) } } - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_monitor_diagnostic_setting" "settings" { From 2270443a607903647dc0c7751e83af098e48715e Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 14:42:04 -0800 Subject: [PATCH 026/119] Add back vars --- .../AksOpenAiTerraform/terraform/main.tf | 66 +------------------ .../AksOpenAiTerraform/terraform/variables.tf | 61 +++++++++++++++++ 2 files changed, 63 insertions(+), 64 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 18f0d9748..6ec202220 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -29,68 +29,6 @@ resource "random_string" "storage_account_suffix" { numeric = false } -variable "name_prefix" { - type = string -} - -variable "log_analytics_workspace_name" { - default = "Workspace" - type = string -} - -variable "log_analytics_retention_days" { - type = number - default = 30 -} - -variable "location" { - default = "westus2" - type = string -} - -variable "resource_group_name" { - default = "RG" - type = string -} - -variable "system_node_pool_subnet_name" { - default = "SystemSubnet" - type = string -} - -variable "user_node_pool_subnet_name" { - default = "UserSubnet" - type = string -} - -variable "pod_subnet_name" { - default = "PodSubnet" - type = string -} - -variable "vm_subnet_name" { - default = "VmSubnet" - type = string -} - -variable "namespace" { - description = "Specifies the namespace of the workload application that accesses the Azure OpenAI Service." - type = string - default = "magic8ball" -} - -variable "service_account_name" { - description = "Specifies the name of the service account of the workload application that accesses the Azure OpenAI Service." - type = string - default = "magic8ball-sa" -} - -variable "email" { - description = "Specifies the email address for the cert-manager cluster issuer." - type = string - default = "paolos@microsoft.com" -} - resource "azurerm_resource_group" "rg" { name = "${var.name_prefix}${var.resource_group_name}" location = var.location @@ -190,11 +128,11 @@ module "container_registry" { module "aks_cluster" { source = "./modules/aks" - name = "${var.name_prefix}${var.aks_cluster_name}" + name = "${var.name_prefix}AksCluster" location = var.location resource_group_name = azurerm_resource_group.rg.name resource_group_id = azurerm_resource_group.rg.id - kubernetes_version = var.kubernetes_version + kubernetes_version = "1.32" sku_tier = "Free" depends_on = [ diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index e69de29bb..38bab3861 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -0,0 +1,61 @@ +variable "name_prefix" { + type = string +} + +variable "log_analytics_workspace_name" { + default = "Workspace" + type = string +} + +variable "log_analytics_retention_days" { + type = number + default = 30 +} + +variable "location" { + default = "westus2" + type = string +} + +variable "resource_group_name" { + default = "RG" + type = string +} + +variable "system_node_pool_subnet_name" { + default = "SystemSubnet" + type = string +} + +variable "user_node_pool_subnet_name" { + default = "UserSubnet" + type = string +} + +variable "pod_subnet_name" { + default = "PodSubnet" + type = string +} + +variable "vm_subnet_name" { + default = "VmSubnet" + type = string +} + +variable "namespace" { + description = "Specifies the namespace of the workload application that accesses the Azure OpenAI Service." + type = string + default = "magic8ball" +} + +variable "service_account_name" { + description = "Specifies the name of the service account of the workload application that accesses the Azure OpenAI Service." + type = string + default = "magic8ball-sa" +} + +variable "email" { + description = "Specifies the email address for the cert-manager cluster issuer." + type = string + default = "paolos@microsoft.com" +} \ No newline at end of file From 4fa3cc873ff3a8cfc78882245de2ae3f74877119 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 14:47:20 -0800 Subject: [PATCH 027/119] More clean up --- .../AksOpenAiTerraform/terraform/main.tf | 30 +++++++++---------- .../modules/private_endpoint/main.tf | 7 ----- .../modules/private_endpoint/outputs.tf | 14 --------- .../terraform/modules/storage_account/main.tf | 6 ---- .../terraform/modules/virtual_network/main.tf | 7 ----- .../modules/virtual_network/outputs.tf | 19 ------------ .../modules/virtual_network/variables.tf | 5 ---- 7 files changed, 15 insertions(+), 73 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 6ec202220..9a0426ce5 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -34,6 +34,21 @@ resource "azurerm_resource_group" "rg" { location = var.location } +module "aks_cluster" { + source = "./modules/aks" + name = "${var.name_prefix}AksCluster" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + resource_group_id = azurerm_resource_group.rg.id + kubernetes_version = "1.32" + sku_tier = "Free" + + depends_on = [ + module.nat_gateway, + module.container_registry + ] +} + module "log_analytics_workspace" { source = "./modules/log_analytics" name = "${var.name_prefix}${var.log_analytics_workspace_name}" @@ -126,21 +141,6 @@ module "container_registry" { admin_enabled = true } -module "aks_cluster" { - source = "./modules/aks" - name = "${var.name_prefix}AksCluster" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - resource_group_id = azurerm_resource_group.rg.id - kubernetes_version = "1.32" - sku_tier = "Free" - - depends_on = [ - module.nat_gateway, - module.container_registry - ] -} - module "openai" { source = "./modules/openai" name = "${var.name_prefix}OpenAi" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf index ae49a166e..2b9b78868 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf @@ -3,7 +3,6 @@ resource "azurerm_private_endpoint" "private_endpoint" { location = var.location resource_group_name = var.resource_group_name subnet_id = var.subnet_id - tags = var.tags private_service_connection { name = "${var.name}Connection" @@ -17,10 +16,4 @@ resource "azurerm_private_endpoint" "private_endpoint" { name = var.private_dns_zone_group_name private_dns_zone_ids = var.private_dns_zone_group_ids } - - lifecycle { - ignore_changes = [ - tags - ] - } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/outputs.tf deleted file mode 100644 index ef51964b0..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/outputs.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "id" { - description = "Specifies the resource id of the private endpoint." - value = azurerm_private_endpoint.private_endpoint.id -} - -output "private_dns_zone_group" { - description = "Specifies the private dns zone group of the private endpoint." - value = azurerm_private_endpoint.private_endpoint.private_dns_zone_group -} - -output "private_dns_zone_configs" { - description = "Specifies the private dns zone(s) configuration" - value = azurerm_private_endpoint.private_endpoint.private_dns_zone_configs -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf index 2cfa39239..a54ed2f26 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf @@ -20,10 +20,4 @@ resource "azurerm_storage_account" "storage_account" { identity { type = "SystemAssigned" } - - lifecycle { - ignore_changes = [ - tags - ] - } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf index bb9443977..72b2c948f 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf @@ -3,13 +3,6 @@ resource "azurerm_virtual_network" "vnet" { address_space = var.address_space location = var.location resource_group_name = var.resource_group_name - tags = var.tags - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_subnet" "subnet" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf deleted file mode 100644 index 4f0e02711..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -output "name" { - description = "Specifies the name of the virtual network" - value = azurerm_virtual_network.vnet.name -} - -output "vnet_id" { - description = "Specifies the resource id of the virtual network" - value = azurerm_virtual_network.vnet.id -} - -output "subnet_ids" { - description = "Contains a list of the the resource id of the subnets" - value = { for subnet in azurerm_subnet.subnet : subnet.name => subnet.id } -} - -output "subnet_ids_as_list" { - description = "Returns the list of the subnet ids as a list of strings." - value = [ for subnet in azurerm_subnet.subnet : subnet.id ] -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf index 02dec85dd..2350dea5b 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf @@ -32,11 +32,6 @@ variable "subnets" { })) } -variable "tags" { - description = "(Optional) Specifies the tags of the storage account" - default = {} -} - variable "log_analytics_workspace_id" { description = "Specifies the log analytics workspace id" type = string From 553de4ed60f6c743a325a275b3640464048e6338 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 15:00:07 -0800 Subject: [PATCH 028/119] More bullshit --- .../AksOpenAiTerraform/terraform/main.tf | 90 +++++++++---------- .../terraform/modules/aks/ssh.tf | 19 ---- .../terraform/modules/bastion_host/main.tf | 1 - .../terraform/modules/bastion_host/output.tf | 23 ----- .../modules/bastion_host/variables.tf | 12 --- .../modules/container_registry/main.tf | 10 --- .../modules/container_registry/outputs.tf | 29 ------ .../modules/container_registry/variables.tf | 25 ------ .../modules/deployment_script/main.tf | 7 -- .../modules/deployment_script/output.tf | 9 -- .../terraform/modules/route_table/main.tf | 8 -- .../modules/storage_account/outputs.tf | 24 ----- .../modules/virtual_network_peering/main.tf | 17 ---- .../virtual_network_peering/variables.tf | 41 --------- 14 files changed, 45 insertions(+), 270 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/output.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/output.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/variables.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 9a0426ce5..b42337b92 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -34,6 +34,28 @@ resource "azurerm_resource_group" "rg" { location = var.location } +module "openai" { + source = "./modules/openai" + name = "${var.name_prefix}OpenAi" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + sku_name = "S0" + deployments = [ + { + name = "gpt-35-turbo" + model = { + name = "gpt-35-turbo" + version = "0301" + } + rai_policy_name = "" + } + ] + custom_subdomain_name = lower("${var.name_prefix}OpenAi") + public_network_access_enabled = true + log_analytics_workspace_id = module.log_analytics_workspace.id + log_analytics_retention_days = var.log_analytics_retention_days +} + module "aks_cluster" { source = "./modules/aks" name = "${var.name_prefix}AksCluster" @@ -49,18 +71,16 @@ module "aks_cluster" { ] } -module "log_analytics_workspace" { - source = "./modules/log_analytics" - name = "${var.name_prefix}${var.log_analytics_workspace_name}" - location = var.location - resource_group_name = azurerm_resource_group.rg.name +module "container_registry" { + source = "./modules/container_registry" + name = "${var.name_prefix}Acr" + location = var.location + resource_group_name = azurerm_resource_group.rg.name - solution_plan_map = { - ContainerInsights= { - product = "OMSGallery/ContainerInsights" - publisher = "Microsoft" - } - } + log_analytics_workspace_id = module.log_analytics_workspace.id + + sku = "Basic" + admin_enabled = true } module "virtual_network" { @@ -129,40 +149,6 @@ module "nat_gateway" { subnet_ids = module.virtual_network.subnet_ids } -module "container_registry" { - source = "./modules/container_registry" - name = "${var.name_prefix}Acr" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - - log_analytics_workspace_id = module.log_analytics_workspace.id - - sku = "Basic" - admin_enabled = true -} - -module "openai" { - source = "./modules/openai" - name = "${var.name_prefix}OpenAi" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - sku_name = "S0" - deployments = [ - { - name = "gpt-35-turbo" - model = { - name = "gpt-35-turbo" - version = "0301" - } - rai_policy_name = "" - } - ] - custom_subdomain_name = lower("${var.name_prefix}OpenAi") - public_network_access_enabled = true - log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = var.log_analytics_retention_days -} - resource "azurerm_user_assigned_identity" "aks_workload_identity" { name = "${var.name_prefix}WorkloadManagedIdentity" resource_group_name = azurerm_resource_group.rg.name @@ -359,3 +345,17 @@ module "deployment_script" { module.aks_cluster ] } + +module "log_analytics_workspace" { + source = "./modules/log_analytics" + name = "${var.name_prefix}${var.log_analytics_workspace_name}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + + solution_plan_map = { + ContainerInsights= { + product = "OMSGallery/ContainerInsights" + publisher = "Microsoft" + } + } +} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf deleted file mode 100644 index 364aa884e..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/ssh.tf +++ /dev/null @@ -1,19 +0,0 @@ -resource "random_pet" "ssh_key_name" { - prefix = "ssh" -} - -resource "azapi_resource_action" "ssh_public_key_gen" { - type = "Microsoft.Compute/sshPublicKeys@2024-07-01" - resource_id = azapi_resource.ssh_public_key.id - action = "generateKeyPair" - method = "POST" - - response_export_values = ["publicKey", "privateKey"] -} - -resource "azapi_resource" "ssh_public_key" { - type = "Microsoft.Compute/sshPublicKeys@2024-07-01" - name = random_pet.ssh_key_name.id - location = var.location - parent_id = var.resource_group_id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf index b7a3ac116..4066b7c17 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf @@ -10,7 +10,6 @@ resource "azurerm_bastion_host" "bastion_host" { name = var.name location = var.location resource_group_name = var.resource_group_name - tags = var.tags ip_configuration { name = "configuration" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/output.tf deleted file mode 100644 index 91b9f9386..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/output.tf +++ /dev/null @@ -1,23 +0,0 @@ -output "name" { - depends_on = [azurerm_bastion_host.bastion_host] - value = azurerm_bastion_host.bastion_host.*.name - description = "Specifies the name of the bastion host" -} - -output "id" { - depends_on = [azurerm_bastion_host.bastion_host] - value = azurerm_bastion_host.bastion_host.*.id - description = "Specifies the resource id of the bastion host" -} - -output "bastion_host" { - depends_on = [azurerm_bastion_host.bastion_host] - value = azurerm_bastion_host.bastion_host - description = "Contains the bastion host resource" -} - -output "public_ip_address" { - depends_on = [azurerm_bastion_host.bastion_host] - value = azurerm_public_ip.public_ip.ip_address - description = "Contains the public IP address of the bastion host." -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf index 77f686eed..e87b7a940 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf @@ -1,35 +1,23 @@ variable "resource_group_name" { - description = "(Required) Specifies the resource group name of the bastion host" type = string } variable "name" { - description = "(Required) Specifies the name of the bastion host" type = string } variable "location" { - description = "(Required) Specifies the location of the bastion host" type = string } -variable "tags" { - description = "(Optional) Specifies the tags of the bastion host" - default = {} -} - variable "subnet_id" { - description = "(Required) Specifies subnet id of the bastion host" type = string } variable "log_analytics_workspace_id" { - description = "Specifies the log analytics workspace id" type = string } variable "log_analytics_retention_days" { - description = "Specifies the number of days of the retention policy" type = number - default = 7 } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf index 32f63c469..546068451 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf @@ -4,7 +4,6 @@ resource "azurerm_container_registry" "acr" { location = var.location sku = var.sku admin_enabled = var.admin_enabled - tags = var.tags identity { type = "UserAssigned" @@ -12,15 +11,6 @@ resource "azurerm_container_registry" "acr" { azurerm_user_assigned_identity.acr_identity.id ] } - - dynamic "georeplications" { - for_each = var.georeplication_locations - - content { - location = georeplications.value - tags = var.tags - } - } } resource "azurerm_user_assigned_identity" "acr_identity" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf deleted file mode 100644 index 1834bc59c..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf +++ /dev/null @@ -1,29 +0,0 @@ -output "name" { - description = "Specifies the name of the container registry." - value = azurerm_container_registry.acr.name -} - -output "id" { - description = "Specifies the resource id of the container registry." - value = azurerm_container_registry.acr.id -} - -output "resource_group_name" { - description = "Specifies the name of the resource group." - value = var.resource_group_name -} - -output "login_server" { - description = "Specifies the login server of the container registry." - value = azurerm_container_registry.acr.login_server -} - -output "login_server_url" { - description = "Specifies the login server url of the container registry." - value = "https://${azurerm_container_registry.acr.login_server}" -} - -output "admin_username" { - description = "Specifies the admin username of the container registry." - value = azurerm_container_registry.acr.admin_username -} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf index 6550f9570..f0f395ad8 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf @@ -1,48 +1,23 @@ variable "name" { - description = "(Required) Specifies the name of the Container Registry. Changing this forces a new resource to be created." type = string } variable "resource_group_name" { - description = "(Required) The name of the resource group in which to create the Container Registry. Changing this forces a new resource to be created." type = string } variable "location" { - description = "(Required) Specifies the supported Azure location where the resource exists. Changing this forces a new resource to be created." type = string } variable "admin_enabled" { - description = "(Optional) Specifies whether the admin user is enabled. Defaults to false." type = string - default = false } variable "sku" { - description = "(Optional) The SKU name of the container registry. Possible values are Basic, Standard and Premium. Defaults to Basic" type = string - default = "Basic" - - validation { - condition = contains(["Basic", "Standard", "Premium"], var.sku) - error_message = "The container registry sku is invalid." - } -} - -variable "tags" { - description = "(Optional) A mapping of tags to assign to the resource." - type = map(any) - default = {} -} - -variable "georeplication_locations" { - description = "(Optional) A list of Azure locations where the container registry should be geo-replicated." - type = list(string) - default = [] } variable "log_analytics_workspace_id" { - description = "Specifies the log analytics workspace id" type = string } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf index e5f05b5f8..4ca1f2e90 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf @@ -2,13 +2,6 @@ resource "azurerm_user_assigned_identity" "script_identity" { name = var.managed_identity_name location = var.location resource_group_name = var.resource_group_name - tags = var.tags - - lifecycle { - ignore_changes = [ - tags - ] - } } data "azurerm_kubernetes_cluster" "aks_cluster" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/output.tf deleted file mode 100644 index 2b3b8e992..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/output.tf +++ /dev/null @@ -1,9 +0,0 @@ -output "id" { - value = azurerm_resource_deployment_script_azure_cli.script.id - description = "Specifies the resource id of the deployment script" -} - -output "outputs" { - value = azurerm_resource_deployment_script_azure_cli.script.outputs - description = "Specifies the list of script outputs." -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf index 0f9a4b649..58971058f 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf @@ -5,7 +5,6 @@ resource "azurerm_route_table" "rt" { name = var.route_table_name location = var.location resource_group_name = var.resource_group_name - tags = var.tags route { name = "kubenetfw_fw_r" @@ -13,13 +12,6 @@ resource "azurerm_route_table" "rt" { next_hop_type = "VirtualAppliance" next_hop_in_ip_address = var.firewall_private_ip } - - lifecycle { - ignore_changes = [ - tags, - route - ] - } } resource "azurerm_subnet_route_table_association" "subnet_association" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf deleted file mode 100644 index c61fdd254..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf +++ /dev/null @@ -1,24 +0,0 @@ -output "name" { - description = "Specifies the name of the storage account" - value = azurerm_storage_account.storage_account.name -} - -output "id" { - description = "Specifies the resource id of the storage account" - value = azurerm_storage_account.storage_account.id -} - -output "primary_access_key" { - description = "Specifies the primary access key of the storage account" - value = azurerm_storage_account.storage_account.primary_access_key -} - -output "principal_id" { - description = "Specifies the principal id of the system assigned managed identity of the storage account" - value = azurerm_storage_account.storage_account.identity[0].principal_id -} - -output "primary_blob_endpoint" { - description = "Specifies the primary blob endpoint of the storage account" - value = azurerm_storage_account.storage_account.primary_blob_endpoint -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/main.tf deleted file mode 100644 index ea60dd098..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/main.tf +++ /dev/null @@ -1,17 +0,0 @@ -resource "azurerm_virtual_network_peering" "peering" { - name = var.peering_name_1_to_2 - resource_group_name = var.vnet_1_rg - virtual_network_name = var.vnet_1_name - remote_virtual_network_id = var.vnet_2_id - allow_virtual_network_access = true - allow_forwarded_traffic = true -} - -resource "azurerm_virtual_network_peering" "peering-back" { - name = var.peering_name_2_to_1 - resource_group_name = var.vnet_2_rg - virtual_network_name = var.vnet_2_name - remote_virtual_network_id = var.vnet_1_id - allow_virtual_network_access = true - allow_forwarded_traffic = true -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/variables.tf deleted file mode 100644 index 9bb640f25..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network_peering/variables.tf +++ /dev/null @@ -1,41 +0,0 @@ -variable "vnet_1_name" { - description = "Specifies the name of the first virtual network" - type = string -} - -variable "vnet_1_id" { - description = "Specifies the resource id of the first virtual network" - type = string -} - -variable "vnet_1_rg" { - description = "Specifies the resource group name of the first virtual network" - type = string -} - -variable "vnet_2_name" { - description = "Specifies the name of the second virtual network" - type = string -} - -variable "vnet_2_id" { - description = "Specifies the resource id of the second virtual network" - type = string -} - -variable "vnet_2_rg" { - description = "Specifies the resource group name of the second virtual network" - type = string -} - -variable "peering_name_1_to_2" { - description = "(Optional) Specifies the name of the first to second virtual network peering" - type = string - default = "peering1to2" -} - -variable "peering_name_2_to_1" { - description = "(Optional) Specifies the name of the second to first virtual network peering" - type = string - default = "peering2to1" -} \ No newline at end of file From ed0d1d195f7fe045c4d37409bc4784d89d945af4 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 15:01:45 -0800 Subject: [PATCH 029/119] Format --- .../AksOpenAiTerraform/terraform/main.tf | 190 +++++++++--------- .../terraform/modules/aks/main.tf | 56 +++--- .../terraform/modules/aks/variables.tf | 38 +++- .../modules/bastion_host/variables.tf | 12 +- .../modules/container_registry/main.tf | 12 +- .../modules/container_registry/variables.tf | 12 +- .../modules/deployment_script/main.tf | 50 ++--- .../modules/deployment_script/variables.tf | 34 ++-- .../modules/diagnostic_setting/main.tf | 8 +- .../modules/diagnostic_setting/outputs.tf | 4 +- .../modules/diagnostic_setting/variables.tf | 2 +- .../terraform/modules/firewall/main.tf | 10 +- .../terraform/modules/firewall/outputs.tf | 2 +- .../terraform/modules/firewall/variables.tf | 6 +- .../terraform/modules/key_vault/main.tf | 8 +- .../terraform/modules/key_vault/outputs.tf | 4 +- .../terraform/modules/key_vault/variables.tf | 18 +- .../terraform/modules/log_analytics/output.tf | 14 +- .../modules/log_analytics/variables.tf | 14 +- .../terraform/modules/nat_gateway/main.tf | 8 +- .../terraform/modules/nat_gateway/output.tf | 6 +- .../modules/nat_gateway/variables.tf | 20 +- .../modules/network_security_group/main.tf | 2 +- .../terraform/modules/node_pool/main.tf | 2 +- .../terraform/modules/node_pool/variables.tf | 78 +++---- .../terraform/modules/openai/main.tf | 2 +- .../terraform/modules/openai/output.tf | 14 +- .../terraform/modules/openai/variables.tf | 24 +-- .../modules/private_endpoint/variables.tf | 6 +- .../modules/storage_account/variables.tf | 28 +-- .../terraform/modules/virtual_network/main.tf | 22 +- .../modules/virtual_network/variables.tf | 16 +- .../AksOpenAiTerraform/terraform/variables.tf | 46 ++--- 33 files changed, 392 insertions(+), 376 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index b42337b92..1fbeff87a 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -26,7 +26,7 @@ resource "random_string" "storage_account_suffix" { special = false lower = true upper = false - numeric = false + numeric = false } resource "azurerm_resource_group" "rg" { @@ -35,36 +35,36 @@ resource "azurerm_resource_group" "rg" { } module "openai" { - source = "./modules/openai" - name = "${var.name_prefix}OpenAi" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - sku_name = "S0" - deployments = [ + source = "./modules/openai" + name = "${var.name_prefix}OpenAi" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + sku_name = "S0" + deployments = [ { name = "gpt-35-turbo" model = { - name = "gpt-35-turbo" + name = "gpt-35-turbo" version = "0301" } rai_policy_name = "" } ] - custom_subdomain_name = lower("${var.name_prefix}OpenAi") - public_network_access_enabled = true - log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = var.log_analytics_retention_days + custom_subdomain_name = lower("${var.name_prefix}OpenAi") + public_network_access_enabled = true + log_analytics_workspace_id = module.log_analytics_workspace.id + log_analytics_retention_days = var.log_analytics_retention_days } module "aks_cluster" { - source = "./modules/aks" - name = "${var.name_prefix}AksCluster" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - resource_group_id = azurerm_resource_group.rg.id - kubernetes_version = "1.32" - sku_tier = "Free" - + source = "./modules/aks" + name = "${var.name_prefix}AksCluster" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + resource_group_id = azurerm_resource_group.rg.id + kubernetes_version = "1.32" + sku_tier = "Free" + depends_on = [ module.nat_gateway, module.container_registry @@ -72,40 +72,40 @@ module "aks_cluster" { } module "container_registry" { - source = "./modules/container_registry" - name = "${var.name_prefix}Acr" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - - log_analytics_workspace_id = module.log_analytics_workspace.id + source = "./modules/container_registry" + name = "${var.name_prefix}Acr" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + + log_analytics_workspace_id = module.log_analytics_workspace.id - sku = "Basic" - admin_enabled = true + sku = "Basic" + admin_enabled = true } module "virtual_network" { - source = "./modules/virtual_network" - vnet_name = "AksVNet" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - - log_analytics_workspace_id = module.log_analytics_workspace.id - - address_space = ["10.0.0.0/8"] + source = "./modules/virtual_network" + vnet_name = "AksVNet" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + + log_analytics_workspace_id = module.log_analytics_workspace.id + + address_space = ["10.0.0.0/8"] subnets = [ { name : var.system_node_pool_subnet_name address_prefixes : ["10.240.0.0/16"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false - delegation: null + delegation : null }, { name : var.user_node_pool_subnet_name address_prefixes : ["10.241.0.0/16"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false - delegation: null + delegation : null }, { name : var.pod_subnet_name @@ -122,31 +122,31 @@ module "virtual_network" { }, { name : var.vm_subnet_name - address_prefixes : ["10.243.1.0/24"] + address_prefixes : ["10.243.1.0/24"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false - delegation: null + delegation : null }, { name : "AzureBastionSubnet" address_prefixes : ["10.243.2.0/24"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false - delegation: null + delegation : null } ] } module "nat_gateway" { - source = "./modules/nat_gateway" - name = "${var.name_prefix}NatGateway" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - - sku_name = "Standard" - idle_timeout_in_minutes = 4 - zones = ["1"] - subnet_ids = module.virtual_network.subnet_ids + source = "./modules/nat_gateway" + name = "${var.name_prefix}NatGateway" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + + sku_name = "Standard" + idle_timeout_in_minutes = 4 + zones = ["1"] + subnet_ids = module.virtual_network.subnet_ids } resource "azurerm_user_assigned_identity" "aks_workload_identity" { @@ -156,9 +156,9 @@ resource "azurerm_user_assigned_identity" "aks_workload_identity" { } resource "azurerm_role_assignment" "cognitive_services_user_assignment" { - scope = module.openai.id - role_definition_name = "Cognitive Services User" - principal_id = azurerm_user_assigned_identity.aks_workload_identity.principal_id + scope = module.openai.id + role_definition_name = "Cognitive Services User" + principal_id = azurerm_user_assigned_identity.aks_workload_identity.principal_id skip_service_principal_aad_check = true } @@ -172,27 +172,27 @@ resource "azurerm_federated_identity_credential" "federated_identity_credential" } resource "azurerm_role_assignment" "network_contributor_assignment" { - scope = azurerm_resource_group.rg.id - role_definition_name = "Network Contributor" - principal_id = module.aks_cluster.aks_identity_principal_id + scope = azurerm_resource_group.rg.id + role_definition_name = "Network Contributor" + principal_id = module.aks_cluster.aks_identity_principal_id skip_service_principal_aad_check = true } resource "azurerm_role_assignment" "acr_pull_assignment" { - role_definition_name = "AcrPull" - scope = module.container_registry.id - principal_id = module.aks_cluster.kubelet_identity_object_id + role_definition_name = "AcrPull" + scope = module.container_registry.id + principal_id = module.aks_cluster.kubelet_identity_object_id skip_service_principal_aad_check = true } module "storage_account" { - source = "./modules/storage_account" - name = "boot${random_string.storage_account_suffix.result}" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - account_kind = "StorageV2" - account_tier = "Standard" - replication_type = "LRS" + source = "./modules/storage_account" + name = "boot${random_string.storage_account_suffix.result}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + account_kind = "StorageV2" + account_tier = "Standard" + replication_type = "LRS" } module "bastion_host" { @@ -225,48 +225,48 @@ module "key_vault" { } module "acr_private_dns_zone" { - source = "./modules/private_dns_zone" - name = "privatelink.azurecr.io" - resource_group_name = azurerm_resource_group.rg.name - virtual_networks_to_link = { + source = "./modules/private_dns_zone" + name = "privatelink.azurecr.io" + resource_group_name = azurerm_resource_group.rg.name + virtual_networks_to_link = { (module.virtual_network.name) = { - subscription_id = data.azurerm_client_config.current.subscription_id + subscription_id = data.azurerm_client_config.current.subscription_id resource_group_name = azurerm_resource_group.rg.name } } } module "openai_private_dns_zone" { - source = "./modules/private_dns_zone" - name = "privatelink.openai.azure.com" - resource_group_name = azurerm_resource_group.rg.name - virtual_networks_to_link = { + source = "./modules/private_dns_zone" + name = "privatelink.openai.azure.com" + resource_group_name = azurerm_resource_group.rg.name + virtual_networks_to_link = { (module.virtual_network.name) = { - subscription_id = data.azurerm_client_config.current.subscription_id + subscription_id = data.azurerm_client_config.current.subscription_id resource_group_name = azurerm_resource_group.rg.name } } } module "key_vault_private_dns_zone" { - source = "./modules/private_dns_zone" - name = "privatelink.vaultcore.azure.net" - resource_group_name = azurerm_resource_group.rg.name - virtual_networks_to_link = { + source = "./modules/private_dns_zone" + name = "privatelink.vaultcore.azure.net" + resource_group_name = azurerm_resource_group.rg.name + virtual_networks_to_link = { (module.virtual_network.name) = { - subscription_id = data.azurerm_client_config.current.subscription_id + subscription_id = data.azurerm_client_config.current.subscription_id resource_group_name = azurerm_resource_group.rg.name } } } module "blob_private_dns_zone" { - source = "./modules/private_dns_zone" - name = "privatelink.blob.core.windows.net" - resource_group_name = azurerm_resource_group.rg.name - virtual_networks_to_link = { + source = "./modules/private_dns_zone" + name = "privatelink.blob.core.windows.net" + resource_group_name = azurerm_resource_group.rg.name + virtual_networks_to_link = { (module.virtual_network.name) = { - subscription_id = data.azurerm_client_config.current.subscription_id + subscription_id = data.azurerm_client_config.current.subscription_id resource_group_name = azurerm_resource_group.rg.name } } @@ -341,19 +341,19 @@ module "deployment_script" { subscription_id = data.azurerm_client_config.current.subscription_id workload_managed_identity_client_id = azurerm_user_assigned_identity.aks_workload_identity.client_id - depends_on = [ + depends_on = [ module.aks_cluster - ] + ] } module "log_analytics_workspace" { - source = "./modules/log_analytics" - name = "${var.name_prefix}${var.log_analytics_workspace_name}" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - - solution_plan_map = { - ContainerInsights= { + source = "./modules/log_analytics" + name = "${var.name_prefix}${var.log_analytics_workspace_name}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + + solution_plan_map = { + ContainerInsights = { product = "OMSGallery/ContainerInsights" publisher = "Microsoft" } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index eb331b3d0..c5e896885 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -1,5 +1,5 @@ resource "azurerm_user_assigned_identity" "aks_identity" { - name = "${var.name}Identity" + name = "${var.name}Identity" resource_group_name = var.resource_group_name location = var.location } @@ -22,26 +22,26 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { http_application_routing_enabled = false default_node_pool { - name = "system" - node_count = 1 - vm_size = var.system_node_pool_vm_size - vnet_subnet_id = module.virtual_network.subnet_ids[var.system_node_pool_subnet_name] - pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] - zones = ["1", "2", "3"] - max_pods = 50 - os_disk_type = "Ephemeral" + name = "system" + node_count = 1 + vm_size = var.system_node_pool_vm_size + vnet_subnet_id = module.virtual_network.subnet_ids[var.system_node_pool_subnet_name] + pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] + zones = ["1", "2", "3"] + max_pods = 50 + os_disk_type = "Ephemeral" } identity { - type = "UserAssigned" + type = "UserAssigned" identity_ids = tolist([azurerm_user_assigned_identity.aks_identity.id]) } network_profile { - dns_service_ip = "10.2.0.10" - network_plugin = "azure" - outbound_type = "userAssignedNATGateway" - service_cidr = "10.2.0.0/24" + dns_service_ip = "10.2.0.10" + network_plugin = "azure" + outbound_type = "userAssignedNATGateway" + service_cidr = "10.2.0.0/24" } oms_agent { @@ -50,8 +50,8 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { } azure_active_directory_role_based_access_control { - tenant_id = data.azurerm_client_config.current.tenant_id - azure_rbac_enabled = true + tenant_id = data.azurerm_client_config.current.tenant_id + azure_rbac_enabled = true } workload_autoscaler_profile { @@ -61,18 +61,18 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { } resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { - kubernetes_cluster_id = azurerm_kubernetes_cluster.aks_cluster.id - name = "user" - vm_size = var.user_node_pool_vm_size - mode = "User" - zones = ["1", "2", "3"] - vnet_subnet_id = module.virtual_network.subnet_ids[var.user_node_pool_subnet_name] - pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] - orchestrator_version = var.kubernetes_version - max_pods = 50 - os_disk_type = "Ephemeral" - os_type = "Linux" - priority = "Regular" + kubernetes_cluster_id = azurerm_kubernetes_cluster.aks_cluster.id + name = "user" + vm_size = var.user_node_pool_vm_size + mode = "User" + zones = ["1", "2", "3"] + vnet_subnet_id = module.virtual_network.subnet_ids[var.user_node_pool_subnet_name] + pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] + orchestrator_version = var.kubernetes_version + max_pods = 50 + os_disk_type = "Ephemeral" + os_type = "Linux" + priority = "Regular" } resource "azurerm_monitor_diagnostic_setting" "settings" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf index 54339d448..1cdba4e09 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf @@ -1,33 +1,49 @@ variable "name" { - type = string + type = string } variable "resource_group_name" { - type = string + type = string } variable "resource_group_id" { - type = string + type = string } variable "location" { - type = string + type = string } variable "kubernetes_version" { - type = string + type = string } -variable sku_tier { - type = string +variable "sku_tier" { + type = string } variable "system_node_pool_vm_size" { - default = "Standard_D8ds_v5" - type = string + default = "Standard_D8ds_v5" + type = string } variable "user_node_pool_vm_size" { - default = "Standard_D8ds_v5" - type = string + default = "Standard_D8ds_v5" + type = string +} + +variable "log_analytics_workspace_id" { + type = string +} + +variable "user_node_pool_subnet_name" { + type = string +} + +variable "system_node_pool_subnet_name" { + type = string +} + +variable "pod_subnet_name" { + type = string } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf index e87b7a940..ab2e33027 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf @@ -1,23 +1,23 @@ variable "resource_group_name" { - type = string + type = string } variable "name" { - type = string + type = string } variable "location" { - type = string + type = string } variable "subnet_id" { - type = string + type = string } variable "log_analytics_workspace_id" { - type = string + type = string } variable "log_analytics_retention_days" { - type = number + type = number } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf index 546068451..52e65bc5d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf @@ -1,9 +1,9 @@ resource "azurerm_container_registry" "acr" { - name = var.name - resource_group_name = var.resource_group_name - location = var.location - sku = var.sku - admin_enabled = var.admin_enabled + name = var.name + resource_group_name = var.resource_group_name + location = var.location + sku = var.sku + admin_enabled = var.admin_enabled identity { type = "UserAssigned" @@ -14,7 +14,7 @@ resource "azurerm_container_registry" "acr" { } resource "azurerm_user_assigned_identity" "acr_identity" { - name = "${var.name}Identity" + name = "${var.name}Identity" resource_group_name = var.resource_group_name location = var.location } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf index f0f395ad8..bf4616efb 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf @@ -1,23 +1,23 @@ variable "name" { - type = string + type = string } variable "resource_group_name" { - type = string + type = string } variable "location" { - type = string + type = string } variable "admin_enabled" { - type = string + type = string } variable "sku" { - type = string + type = string } variable "log_analytics_workspace_id" { - type = string + type = string } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf index 4ca1f2e90..38e5cc841 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf @@ -6,13 +6,13 @@ resource "azurerm_user_assigned_identity" "script_identity" { data "azurerm_kubernetes_cluster" "aks_cluster" { name = var.aks_cluster_name - resource_group_name = var.resource_group_name + resource_group_name = var.resource_group_name } resource "azurerm_role_assignment" "network_contributor_assignment" { - scope = data.azurerm_kubernetes_cluster.aks_cluster.id - role_definition_name = "Azure Kubernetes Service Cluster Admin Role" - principal_id = azurerm_user_assigned_identity.script_identity.principal_id + scope = data.azurerm_kubernetes_cluster.aks_cluster.id + role_definition_name = "Azure Kubernetes Service Cluster Admin Role" + principal_id = azurerm_user_assigned_identity.script_identity.principal_id skip_service_principal_aad_check = true } @@ -31,58 +31,58 @@ resource "azurerm_resource_deployment_script_azure_cli" "script" { identity { type = "UserAssigned" - identity_ids = [ + identity_ids = [ azurerm_user_assigned_identity.script_identity.id ] } environment_variable { - name = "clusterName" - value = var.aks_cluster_name + name = "clusterName" + value = var.aks_cluster_name } environment_variable { - name = "resourceGroupName" - value = var.resource_group_name + name = "resourceGroupName" + value = var.resource_group_name } environment_variable { - name = "applicationGatewayEnabled" - value = false + name = "applicationGatewayEnabled" + value = false } environment_variable { - name = "tenantId" - value = var.tenant_id + name = "tenantId" + value = var.tenant_id } environment_variable { - name = "subscriptionId" - value = var.subscription_id + name = "subscriptionId" + value = var.subscription_id } environment_variable { - name = "hostName" - value = var.hostname + name = "hostName" + value = var.hostname } environment_variable { - name = "namespace" - value = var.namespace + name = "namespace" + value = var.namespace } environment_variable { - name = "serviceAccountName" - value = var.service_account_name + name = "serviceAccountName" + value = var.service_account_name } environment_variable { - name = "workloadManagedIdentityClientId" - value = var.workload_managed_identity_client_id + name = "workloadManagedIdentityClientId" + value = var.workload_managed_identity_client_id } environment_variable { - name = "email" - value = var.email + name = "email" + value = var.email } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf index ca7442247..f650b86fc 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf @@ -1,74 +1,74 @@ variable "resource_group_name" { description = "(Required) Specifies the resource group name" - type = string + type = string } variable "location" { description = "(Required) Specifies the location of the Azure OpenAI Service" - type = string + type = string } variable "name" { description = "(Required) Specifies the name of the Azure OpenAI Service" - type = string - default = "BashScript" + type = string + default = "BashScript" } variable "azure_cli_version" { description = "(Required) Azure CLI module version to be used." - type = string - default = "2.9.1" + type = string + default = "2.9.1" } variable "managed_identity_name" { description = "Specifies the name of the user-defined managed identity used by the deployment script." - type = string - default = "ScriptManagedIdentity" + type = string + default = "ScriptManagedIdentity" } variable "primary_script_uri" { description = "(Optional) Uri for the script. This is the entry point for the external script. Changing this forces a new Resource Deployment Script to be created." - type = string + type = string } variable "aks_cluster_name" { description = "Specifies the name of the AKS cluster." - type = string + type = string } variable "tenant_id" { description = "Specifies the Azure AD tenant id." - type = string + type = string } variable "subscription_id" { description = "Specifies the Azure subscription id." - type = string + type = string } variable "hostname" { description = "Specifies the hostname of the application." - type = string + type = string } variable "namespace" { description = "Specifies the namespace of the application." - type = string + type = string } variable "service_account_name" { description = "Specifies the service account of the application." - type = string + type = string } variable "workload_managed_identity_client_id" { description = "Specifies the client id of the workload user-defined managed identity." - type = string + type = string } variable "email" { description = "Specifies the email address for the cert-manager cluster issuer." - type = string + type = string } variable "tags" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf index 45d29f614..3f8f5af32 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf @@ -1,14 +1,14 @@ resource "azurerm_monitor_diagnostic_setting" "settings" { - name = var.name - target_resource_id = var.target_resource_id + name = var.name + target_resource_id = var.target_resource_id log_analytics_workspace_id = var.log_analytics_workspace_id log_analytics_destination_type = var.log_analytics_destination_type eventhub_name = var.eventhub_name - eventhub_authorization_rule_id = var.eventhub_authorization_rule_id + eventhub_authorization_rule_id = var.eventhub_authorization_rule_id - storage_account_id = var.storage_account_id + storage_account_id = var.storage_account_id dynamic "log" { for_each = toset(logs) diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf index 3b15757f8..3d727607e 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf @@ -1,9 +1,9 @@ output "name" { - value = azurerm_key_vault.key_vault.name + value = azurerm_key_vault.key_vault.name description = "Specifies the name of the key vault." } output "id" { - value = azurerm_key_vault.key_vault.id + value = azurerm_key_vault.key_vault.id description = "Specifies the resource id of the key vault." } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf index 5fefdb86a..7165884e9 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf @@ -33,7 +33,7 @@ variable "target_resource_id" { variable "log_analytics_workspace_id" { description = "(Optional) Specifies the ID of a Log Analytics Workspace where Diagnostics Data should be sent." - type = string + type = string } variable "log_analytics_destination_type" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf index 0d5474863..3ce12243d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf @@ -91,7 +91,7 @@ resource "azurerm_firewall_policy_rule_collection_group" "policy" { type = "Https" } } - + rule { name = "AllowImagesFqdns" source_addresses = ["*"] @@ -174,15 +174,15 @@ resource "azurerm_firewall_policy_rule_collection_group" "policy" { } rule { - name = "ServiceTags" - source_addresses = ["*"] - destination_ports = ["*"] + name = "ServiceTags" + source_addresses = ["*"] + destination_ports = ["*"] destination_addresses = [ "AzureContainerRegistry", "MicrosoftContainerRegistry", "AzureActiveDirectory" ] - protocols = ["Any"] + protocols = ["Any"] } rule { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf index b11aab5ea..f280bb2c1 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf @@ -1,4 +1,4 @@ output "private_ip_address" { description = "Specifies the private IP address of the firewall." - value = azurerm_firewall.firewall.ip_configuration[0].private_ip_address + value = azurerm_firewall.firewall.ip_configuration[0].private_ip_address } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf index dedd9481b..aa67baa3b 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf @@ -9,7 +9,7 @@ variable "sku_name" { type = string validation { - condition = contains(["AZFW_Hub", "AZFW_VNet" ], var.sku_name) + condition = contains(["AZFW_Hub", "AZFW_VNet"], var.sku_name) error_message = "The value of the sku name property of the firewall is invalid." } } @@ -20,7 +20,7 @@ variable "sku_tier" { type = string validation { - condition = contains(["Premium", "Standard", "Basic" ], var.sku_tier) + condition = contains(["Premium", "Standard", "Basic"], var.sku_tier) error_message = "The value of the sku tier property of the firewall is invalid." } } @@ -41,7 +41,7 @@ variable "threat_intel_mode" { type = string validation { - condition = contains(["Off", "Alert", "Deny"], var.threat_intel_mode) + condition = contains(["Off", "Alert", "Deny"], var.threat_intel_mode) error_message = "The threat intel mode is invalid." } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf index 0f3f899b6..02cce3be0 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf @@ -11,7 +11,7 @@ resource "azurerm_key_vault" "key_vault" { enable_rbac_authorization = var.enable_rbac_authorization purge_protection_enabled = var.purge_protection_enabled soft_delete_retention_days = var.soft_delete_retention_days - + timeouts { delete = "60m" } @@ -24,9 +24,9 @@ resource "azurerm_key_vault" "key_vault" { } lifecycle { - ignore_changes = [ - tags - ] + ignore_changes = [ + tags + ] } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf index 3b15757f8..3d727607e 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf @@ -1,9 +1,9 @@ output "name" { - value = azurerm_key_vault.key_vault.name + value = azurerm_key_vault.key_vault.name description = "Specifies the name of the key vault." } output "id" { - value = azurerm_key_vault.key_vault.id + value = azurerm_key_vault.key_vault.id description = "Specifies the resource id of the key vault." } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf index df4cdbe55..628c6bdbc 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf @@ -24,7 +24,7 @@ variable "sku_name" { default = "standard" validation { - condition = contains(["standard", "premium" ], var.sku_name) + condition = contains(["standard", "premium"], var.sku_name) error_message = "The value of the sku name property of the key vault is invalid." } } @@ -71,34 +71,34 @@ variable "soft_delete_retention_days" { default = 30 } -variable "bypass" { +variable "bypass" { description = "(Required) Specifies which traffic can bypass the network rules. Possible values are AzureServices and None." type = string - default = "AzureServices" + default = "AzureServices" validation { - condition = contains(["AzureServices", "None" ], var.bypass) + condition = contains(["AzureServices", "None"], var.bypass) error_message = "The valut of the bypass property of the key vault is invalid." } } -variable "default_action" { +variable "default_action" { description = "(Required) The Default Action to use when no rules match from ip_rules / virtual_network_subnet_ids. Possible values are Allow and Deny." type = string - default = "Allow" + default = "Allow" validation { - condition = contains(["Allow", "Deny" ], var.default_action) + condition = contains(["Allow", "Deny"], var.default_action) error_message = "The value of the default action property of the key vault is invalid." } } -variable "ip_rules" { +variable "ip_rules" { description = "(Optional) One or more IP Addresses, or CIDR Blocks which should be able to access the Key Vault." default = [] } -variable "virtual_network_subnet_ids" { +variable "virtual_network_subnet_ids" { description = "(Optional) One or more Subnet ID's which should be able to access this Key Vault." default = [] } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf index 8cb42544a..7abcf881f 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf @@ -1,30 +1,30 @@ output "id" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.id + value = azurerm_log_analytics_workspace.log_analytics_workspace.id description = "Specifies the resource id of the log analytics workspace" } output "location" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.location + value = azurerm_log_analytics_workspace.log_analytics_workspace.location description = "Specifies the location of the log analytics workspace" } output "name" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.name + value = azurerm_log_analytics_workspace.log_analytics_workspace.name description = "Specifies the name of the log analytics workspace" } output "resource_group_name" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.resource_group_name + value = azurerm_log_analytics_workspace.log_analytics_workspace.resource_group_name description = "Specifies the name of the resource group that contains the log analytics workspace" } output "workspace_id" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.workspace_id + value = azurerm_log_analytics_workspace.log_analytics_workspace.workspace_id description = "Specifies the workspace id of the log analytics workspace" } output "primary_shared_key" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.primary_shared_key + value = azurerm_log_analytics_workspace.log_analytics_workspace.primary_shared_key description = "Specifies the workspace key of the log analytics workspace" - sensitive = true + sensitive = true } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf index d6226a996..ed214b0b1 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf @@ -1,25 +1,25 @@ variable "resource_group_name" { description = "(Required) Specifies the resource group name" - type = string + type = string } variable "location" { description = "(Required) Specifies the location of the log analytics workspace" - type = string + type = string } variable "name" { description = "(Required) Specifies the name of the log analytics workspace" - type = string + type = string } variable "sku" { description = "(Optional) Specifies the sku of the log analytics workspace" - type = string - default = "PerGB2018" - + type = string + default = "PerGB2018" + validation { - condition = contains(["Free", "Standalone", "PerNode", "PerGB2018"], var.sku) + condition = contains(["Free", "Standalone", "PerNode", "PerGB2018"], var.sku) error_message = "The log analytics sku is incorrect." } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf index 74e201a8c..bb5d7c5b0 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf @@ -5,7 +5,7 @@ resource "azurerm_public_ip" "nat_gategay_public_ip" { allocation_method = "Static" sku = "Standard" zones = var.zones - tags = var.tags + tags = var.tags lifecycle { ignore_changes = [ @@ -36,7 +36,7 @@ resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_asso } resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { - for_each = var.subnet_ids - subnet_id = each.value - nat_gateway_id = azurerm_nat_gateway.nat_gateway.id + for_each = var.subnet_ids + subnet_id = each.value + nat_gateway_id = azurerm_nat_gateway.nat_gateway.id } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf index 014ece6b0..2b9ce3bb5 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf @@ -1,14 +1,14 @@ output "name" { - value = azurerm_nat_gateway.nat_gateway.name + value = azurerm_nat_gateway.nat_gateway.name description = "Specifies the name of the Azure NAT Gateway" } output "id" { - value = azurerm_nat_gateway.nat_gateway.id + value = azurerm_nat_gateway.nat_gateway.id description = "Specifies the resource id of the Azure NAT Gateway" } output "public_ip_address" { - value = azurerm_public_ip.nat_gategay_public_ip.ip_address + value = azurerm_public_ip.nat_gategay_public_ip.ip_address description = "Contains the public IP address of the Azure NAT Gateway." } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf index 0e11ddadc..14f745663 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf @@ -1,16 +1,16 @@ variable "resource_group_name" { description = "(Required) Specifies the resource group name" - type = string + type = string } variable "location" { description = "(Required) Specifies the location of the Azure OpenAI Service" - type = string + type = string } variable "name" { description = "(Required) Specifies the name of the Azure OpenAI Service" - type = string + type = string } variable "tags" { @@ -21,23 +21,23 @@ variable "tags" { variable "sku_name" { description = "(Optional) The SKU which should be used. At this time the only supported value is Standard. Defaults to Standard" - type = string - default = "Standard" + type = string + default = "Standard" } variable "idle_timeout_in_minutes" { description = "(Optional) The idle timeout which should be used in minutes. Defaults to 4." - type = number - default = 4 + type = number + default = 4 } variable "zones" { description = " (Optional) A list of Availability Zones in which this NAT Gateway should be located. Changing this forces a new NAT Gateway to be created." - type = list(string) - default = [] + type = list(string) + default = [] } variable "subnet_ids" { description = "(Required) A map of subnet ids to associate with the NAT Gateway" - type = map(string) + type = map(string) } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf index ba9652eb2..be9f9cbf2 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf @@ -35,7 +35,7 @@ resource "azurerm_monitor_diagnostic_setting" "settings" { category = "NetworkSecurityGroupEvent" } - enabled_log { + enabled_log { category = "NetworkSecurityGroupRuleCounter" } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf index acdeda9c3..a28e1582e 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf @@ -19,7 +19,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { lifecycle { ignore_changes = [ - tags + tags ] } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf index b95bf813f..2e2825bd6 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf @@ -16,43 +16,43 @@ variable "vm_size" { variable "availability_zones" { description = "(Optional) A list of Availability Zones where the Nodes in this Node Pool should be created in. Changing this forces a new resource to be created." type = list(string) - default = ["1", "2", "3"] + default = ["1", "2", "3"] } variable "enable_host_encryption" { description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." - type = bool - default = false -} + type = bool + default = false +} variable "enable_node_public_ip" { description = "(Optional) Should each node have a Public IP Address? Defaults to false. Changing this forces a new resource to be created." - type = bool - default = false -} + type = bool + default = false +} variable "max_pods" { description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." - type = number - default = 250 + type = number + default = 250 } variable "mode" { description = "(Optional) Should this Node Pool be used for System or User resources? Possible values are System and User. Defaults to User." - type = string - default = "User" -} + type = string + default = "User" +} variable "node_labels" { description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." - type = map(any) - default = {} -} + type = map(any) + default = {} +} variable "node_taints" { description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." - type = list(string) - default = [] + type = list(string) + default = [] } variable "tags" { @@ -62,52 +62,52 @@ variable "tags" { variable "orchestrator_version" { description = "(Required) Version of Kubernetes used for the Agents. If not specified, the latest recommended version will be used at provisioning time (but won't auto-upgrade)" - type = string -} + type = string +} variable "os_disk_size_gb" { description = "(Optional) The Agent Operating System disk size in GB. Changing this forces a new resource to be created." - type = number - default = null -} + type = number + default = null +} variable "os_disk_type" { description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." - type = string - default = "Ephemeral" -} + type = string + default = "Ephemeral" +} variable "os_type" { description = "(Optional) The Operating System which should be used for this Node Pool. Changing this forces a new resource to be created. Possible values are Linux and Windows. Defaults to Linux." - type = string - default = "Linux" -} + type = string + default = "Linux" +} variable "priority" { description = "(Optional) The Priority for Virtual Machines within the Virtual Machine Scale Set that powers this Node Pool. Possible values are Regular and Spot. Defaults to Regular. Changing this forces a new resource to be created." - type = string - default = "Regular" -} + type = string + default = "Regular" +} variable "proximity_placement_group_id" { description = "(Optional) The ID of the Proximity Placement Group where the Virtual Machine Scale Set that powers this Node Pool will be placed. Changing this forces a new resource to be created." - type = string - default = null -} + type = string + default = null +} variable "vnet_subnet_id" { description = "(Optional) The ID of the Subnet where this Node Pool should exist." - type = string - default = null + type = string + default = null } variable "pod_subnet_id" { description = "(Optional) The ID of the Subnet where the pods in the system node pool should exist. Changing this forces a new resource to be created." - type = string - default = null + type = string + default = null } -variable resource_group_name { +variable "resource_group_name" { description = "Specifies the resource group name" type = string } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf index 55d6d49c7..235dca40d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf @@ -20,7 +20,7 @@ resource "azurerm_cognitive_account" "openai" { } resource "azurerm_cognitive_deployment" "deployment" { - for_each = {for deployment in var.deployments: deployment.name => deployment} + for_each = { for deployment in var.deployments : deployment.name => deployment } name = each.key cognitive_account_id = azurerm_cognitive_account.openai.id diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf index 85097ba3d..2b3e7cb0c 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf @@ -1,34 +1,34 @@ output "id" { - value = azurerm_cognitive_account.openai.id + value = azurerm_cognitive_account.openai.id description = "Specifies the resource id of the log analytics workspace" } output "location" { - value = azurerm_cognitive_account.openai.location + value = azurerm_cognitive_account.openai.location description = "Specifies the location of the log analytics workspace" } output "name" { - value = azurerm_cognitive_account.openai.name + value = azurerm_cognitive_account.openai.name description = "Specifies the name of the log analytics workspace" } output "resource_group_name" { - value = azurerm_cognitive_account.openai.resource_group_name + value = azurerm_cognitive_account.openai.resource_group_name description = "Specifies the name of the resource group that contains the log analytics workspace" } output "endpoint" { - value = azurerm_cognitive_account.openai.endpoint + value = azurerm_cognitive_account.openai.endpoint description = "Specifies the endpoint of the Azure OpenAI Service." } output "primary_access_key" { - value = azurerm_cognitive_account.openai.endpoint + value = azurerm_cognitive_account.openai.endpoint description = "Specifies the primary access key of the Azure OpenAI Service." } output "secondary_access_key" { - value = azurerm_cognitive_account.openai.endpoint + value = azurerm_cognitive_account.openai.endpoint description = "Specifies the secondary access key of the Azure OpenAI Service." } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf index 1d13d78a6..dca286ff8 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf @@ -1,22 +1,22 @@ variable "resource_group_name" { description = "(Required) Specifies the resource group name" - type = string + type = string } variable "location" { description = "(Required) Specifies the location of the Azure OpenAI Service" - type = string + type = string } variable "name" { description = "(Required) Specifies the name of the Azure OpenAI Service" - type = string + type = string } variable "sku_name" { description = "(Optional) Specifies the sku name for the Azure OpenAI Service" - type = string - default = "S0" + type = string + default = "S0" } variable "tags" { @@ -27,13 +27,13 @@ variable "tags" { variable "custom_subdomain_name" { description = "(Optional) Specifies the custom subdomain name of the Azure OpenAI Service" - type = string + type = string } variable "public_network_access_enabled" { description = "(Optional) Specifies whether public network access is allowed for the Azure OpenAI Service" - type = bool - default = true + type = bool + default = true } variable "deployments" { @@ -41,21 +41,21 @@ variable "deployments" { type = list(object({ name = string model = object({ - name = string + name = string version = string }) - rai_policy_name = string + rai_policy_name = string })) default = [ { name = "gpt-35-turbo" model = { - name = "gpt-35-turbo" + name = "gpt-35-turbo" version = "0301" } rai_policy_name = "" } - ] + ] } variable "log_analytics_workspace_id" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf index 5d9c44048..f7a410572 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf @@ -10,7 +10,7 @@ variable "resource_group_name" { variable "private_connection_resource_id" { description = "(Required) Specifies the resource id of the private link service" - type = string + type = string } variable "location" { @@ -26,7 +26,7 @@ variable "subnet_id" { variable "is_manual_connection" { description = "(Optional) Specifies whether the private endpoint connection requires manual approval from the remote resource owner." type = string - default = false + default = false } variable "subresource_name" { @@ -38,7 +38,7 @@ variable "subresource_name" { variable "request_message" { description = "(Optional) Specifies a message passed to the owner of the remote resource when the private endpoint attempts to establish the connection to the remote resource." type = string - default = null + default = null } variable "private_dns_zone_group_name" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf index 5122b841c..b38fcad5a 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf @@ -18,8 +18,8 @@ variable "account_kind" { default = "StorageV2" type = string - validation { - condition = contains(["Storage", "StorageV2"], var.account_kind) + validation { + condition = contains(["Storage", "StorageV2"], var.account_kind) error_message = "The account kind of the storage account is invalid." } } @@ -29,8 +29,8 @@ variable "account_tier" { default = "Standard" type = string - validation { - condition = contains(["Standard", "Premium"], var.account_tier) + validation { + condition = contains(["Standard", "Premium"], var.account_tier) error_message = "The account tier of the storage account is invalid." } } @@ -41,7 +41,7 @@ variable "replication_type" { type = string validation { - condition = contains(["LRS", "ZRS", "GRS", "GZRS", "RA-GRS", "RA-GZRS"], var.replication_type) + condition = contains(["LRS", "ZRS", "GRS", "GZRS", "RA-GRS", "RA-GZRS"], var.replication_type) error_message = "The replication type of the storage account is invalid." } } @@ -53,21 +53,21 @@ variable "is_hns_enabled" { } variable "default_action" { - description = "Allow or disallow public access to all blobs or containers in the storage accounts. The default interpretation is true for this property." - default = "Allow" - type = string + description = "Allow or disallow public access to all blobs or containers in the storage accounts. The default interpretation is true for this property." + default = "Allow" + type = string } variable "ip_rules" { - description = "Specifies IP rules for the storage account" - default = [] - type = list(string) + description = "Specifies IP rules for the storage account" + default = [] + type = list(string) } variable "virtual_network_subnet_ids" { - description = "Specifies a list of resource ids for subnets" - default = [] - type = list(string) + description = "Specifies a list of resource ids for subnets" + default = [] + type = list(string) } variable "kind" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf index 72b2c948f..879aad9c4 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf @@ -8,21 +8,21 @@ resource "azurerm_virtual_network" "vnet" { resource "azurerm_subnet" "subnet" { for_each = { for subnet in var.subnets : subnet.name => subnet } - name = each.key - resource_group_name = var.resource_group_name - virtual_network_name = azurerm_virtual_network.vnet.name - address_prefixes = each.value.address_prefixes - private_endpoint_network_policies = each.value.private_endpoint_network_policies - private_link_service_network_policies_enabled = each.value.private_link_service_network_policies_enabled + name = each.key + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = each.value.address_prefixes + private_endpoint_network_policies = each.value.private_endpoint_network_policies + private_link_service_network_policies_enabled = each.value.private_link_service_network_policies_enabled dynamic "delegation" { - for_each = each.value.delegation != null ? [each.value.delegation] : [] + for_each = each.value.delegation != null ? [each.value.delegation] : [] content { - name = "delegation" - + name = "delegation" + service_delegation { - name = delegation.value.service_delegation.name - actions = delegation.value.service_delegation.actions + name = delegation.value.service_delegation.name + actions = delegation.value.service_delegation.actions } } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf index 2350dea5b..1e37598b1 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf @@ -21,14 +21,14 @@ variable "address_space" { variable "subnets" { description = "Subnets configuration" type = list(object({ - name = string - address_prefixes = list(string) - private_endpoint_network_policies = string - private_link_service_network_policies_enabled = bool - delegation = object({name = string, service_delegation = object({ - name = string - actions = list(string) - })}) + name = string + address_prefixes = list(string) + private_endpoint_network_policies = string + private_link_service_network_policies_enabled = bool + delegation = object({ name = string, service_delegation = object({ + name = string + actions = list(string) + }) }) })) } diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 38bab3861..aeb7ff5aa 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -1,61 +1,61 @@ variable "name_prefix" { - type = string + type = string } variable "log_analytics_workspace_name" { - default = "Workspace" - type = string + default = "Workspace" + type = string } variable "log_analytics_retention_days" { - type = number - default = 30 + type = number + default = 30 } variable "location" { - default = "westus2" - type = string + default = "westus2" + type = string } variable "resource_group_name" { - default = "RG" - type = string + default = "RG" + type = string } variable "system_node_pool_subnet_name" { - default = "SystemSubnet" - type = string + default = "SystemSubnet" + type = string } variable "user_node_pool_subnet_name" { - default = "UserSubnet" - type = string + default = "UserSubnet" + type = string } variable "pod_subnet_name" { - default = "PodSubnet" - type = string + default = "PodSubnet" + type = string } variable "vm_subnet_name" { - default = "VmSubnet" - type = string + default = "VmSubnet" + type = string } variable "namespace" { description = "Specifies the namespace of the workload application that accesses the Azure OpenAI Service." - type = string - default = "magic8ball" + type = string + default = "magic8ball" } variable "service_account_name" { description = "Specifies the name of the service account of the workload application that accesses the Azure OpenAI Service." - type = string - default = "magic8ball-sa" + type = string + default = "magic8ball-sa" } variable "email" { description = "Specifies the email address for the cert-manager cluster issuer." - type = string - default = "paolos@microsoft.com" + type = string + default = "paolos@microsoft.com" } \ No newline at end of file From 2f4aca49b6d7768100cb97167d7ed9297bb5b6bb Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 15:03:41 -0800 Subject: [PATCH 030/119] Fix --- scenarios/AksOpenAiTerraform/terraform/main.tf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 1fbeff87a..ce7ec690e 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -62,8 +62,14 @@ module "aks_cluster" { location = var.location resource_group_name = azurerm_resource_group.rg.name resource_group_id = azurerm_resource_group.rg.id - kubernetes_version = "1.32" - sku_tier = "Free" + + kubernetes_version = "1.32" + sku_tier = "Free" + user_node_pool_subnet_name = var.user_node_pool_subnet_name + system_node_pool_subnet_name = var.system_node_pool_subnet_name + pod_subnet_name = var.pod_subnet_name + + log_analytics_workspace_id = module.log_analytics_workspace.id depends_on = [ module.nat_gateway, From f1887306eff914beb165f47807f6be2d58e27c98 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 15:13:18 -0800 Subject: [PATCH 031/119] Reorganize --- .../AksOpenAiTerraform/terraform/main.tf | 205 ++++++++++-------- 1 file changed, 110 insertions(+), 95 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index ce7ec690e..f08c2037d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -34,6 +34,9 @@ resource "azurerm_resource_group" "rg" { location = var.location } +############################################################################### +# Application +############################################################################### module "openai" { source = "./modules/openai" name = "${var.name_prefix}OpenAi" @@ -89,6 +92,74 @@ module "container_registry" { admin_enabled = true } +module "storage_account" { + source = "./modules/storage_account" + name = "boot${random_string.storage_account_suffix.result}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + account_kind = "StorageV2" + account_tier = "Standard" + replication_type = "LRS" +} + +module "key_vault" { + source = "./modules/key_vault" + name = "${var.name_prefix}KeyVault" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + tenant_id = data.azurerm_client_config.current.tenant_id + sku_name = "standard" + enabled_for_deployment = true + enabled_for_disk_encryption = true + enabled_for_template_deployment = true + enable_rbac_authorization = true + purge_protection_enabled = false + soft_delete_retention_days = 30 + bypass = "AzureServices" + default_action = "Allow" + log_analytics_workspace_id = module.log_analytics_workspace.id + log_analytics_retention_days = var.log_analytics_retention_days +} + +module "deployment_script" { + source = "./modules/deployment_script" + name = "${var.name_prefix}BashScript" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + azure_cli_version = "2.9.1" + managed_identity_name = "${var.name_prefix}ScriptManagedIdentity" + aks_cluster_name = module.aks_cluster.name + hostname = "magic8ball.contoso.com" + namespace = var.namespace + service_account_name = var.service_account_name + email = var.email + primary_script_uri = "https://paolosalvatori.blob.core.windows.net/scripts/install-nginx-via-helm-and-create-sa.sh" + tenant_id = data.azurerm_client_config.current.tenant_id + subscription_id = data.azurerm_client_config.current.subscription_id + workload_managed_identity_client_id = azurerm_user_assigned_identity.aks_workload_identity.client_id + + depends_on = [ + module.aks_cluster + ] +} + +module "log_analytics_workspace" { + source = "./modules/log_analytics" + name = "${var.name_prefix}${var.log_analytics_workspace_name}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + + solution_plan_map = { + ContainerInsights = { + product = "OMSGallery/ContainerInsights" + publisher = "Microsoft" + } + } +} + +############################################################################### +# Networking +############################################################################### module "virtual_network" { source = "./modules/virtual_network" vnet_name = "AksVNet" @@ -155,52 +226,6 @@ module "nat_gateway" { subnet_ids = module.virtual_network.subnet_ids } -resource "azurerm_user_assigned_identity" "aks_workload_identity" { - name = "${var.name_prefix}WorkloadManagedIdentity" - resource_group_name = azurerm_resource_group.rg.name - location = var.location -} - -resource "azurerm_role_assignment" "cognitive_services_user_assignment" { - scope = module.openai.id - role_definition_name = "Cognitive Services User" - principal_id = azurerm_user_assigned_identity.aks_workload_identity.principal_id - skip_service_principal_aad_check = true -} - -resource "azurerm_federated_identity_credential" "federated_identity_credential" { - name = "${title(var.namespace)}FederatedIdentity" - resource_group_name = azurerm_resource_group.rg.name - audience = ["api://AzureADTokenExchange"] - issuer = module.aks_cluster.oidc_issuer_url - parent_id = azurerm_user_assigned_identity.aks_workload_identity.id - subject = "system:serviceaccount:${var.namespace}:${var.service_account_name}" -} - -resource "azurerm_role_assignment" "network_contributor_assignment" { - scope = azurerm_resource_group.rg.id - role_definition_name = "Network Contributor" - principal_id = module.aks_cluster.aks_identity_principal_id - skip_service_principal_aad_check = true -} - -resource "azurerm_role_assignment" "acr_pull_assignment" { - role_definition_name = "AcrPull" - scope = module.container_registry.id - principal_id = module.aks_cluster.kubelet_identity_object_id - skip_service_principal_aad_check = true -} - -module "storage_account" { - source = "./modules/storage_account" - name = "boot${random_string.storage_account_suffix.result}" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - account_kind = "StorageV2" - account_tier = "Standard" - replication_type = "LRS" -} - module "bastion_host" { source = "./modules/bastion_host" name = "${var.name_prefix}BastionHost" @@ -211,25 +236,9 @@ module "bastion_host" { log_analytics_retention_days = var.log_analytics_retention_days } -module "key_vault" { - source = "./modules/key_vault" - name = "${var.name_prefix}KeyVault" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - tenant_id = data.azurerm_client_config.current.tenant_id - sku_name = "standard" - enabled_for_deployment = true - enabled_for_disk_encryption = true - enabled_for_template_deployment = true - enable_rbac_authorization = true - purge_protection_enabled = false - soft_delete_retention_days = 30 - bypass = "AzureServices" - default_action = "Allow" - log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = var.log_analytics_retention_days -} - +############################################################################### +# Private DNS Zones +############################################################################### module "acr_private_dns_zone" { source = "./modules/private_dns_zone" name = "privatelink.azurecr.io" @@ -278,6 +287,9 @@ module "blob_private_dns_zone" { } } +############################################################################### +# Private Endpoints +############################################################################### module "openai_private_endpoint" { source = "./modules/private_endpoint" name = "${module.openai.name}PrivateEndpoint" @@ -319,7 +331,7 @@ module "key_vault_private_endpoint" { module "blob_private_endpoint" { source = "./modules/private_endpoint" - name = var.name_prefix == null ? "${random_string.prefix.result}BlocStoragePrivateEndpoint" : "${var.name_prefix}BlobStoragePrivateEndpoint" + name = "${var.name_prefix}BlobStoragePrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] @@ -330,38 +342,41 @@ module "blob_private_endpoint" { private_dns_zone_group_ids = [module.blob_private_dns_zone.id] } -module "deployment_script" { - source = "./modules/deployment_script" - name = "${var.name_prefix}BashScript" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - azure_cli_version = "2.9.1" - managed_identity_name = "${var.name_prefix}ScriptManagedIdentity" - aks_cluster_name = module.aks_cluster.name - hostname = "magic8ball.contoso.com" - namespace = var.namespace - service_account_name = var.service_account_name - email = var.email - primary_script_uri = "https://paolosalvatori.blob.core.windows.net/scripts/install-nginx-via-helm-and-create-sa.sh" - tenant_id = data.azurerm_client_config.current.tenant_id - subscription_id = data.azurerm_client_config.current.subscription_id - workload_managed_identity_client_id = azurerm_user_assigned_identity.aks_workload_identity.client_id +############################################################################### +# Identities +############################################################################### +resource "azurerm_user_assigned_identity" "aks_workload_identity" { + name = "${var.name_prefix}WorkloadManagedIdentity" + resource_group_name = azurerm_resource_group.rg.name + location = var.location +} - depends_on = [ - module.aks_cluster - ] +resource "azurerm_role_assignment" "cognitive_services_user_assignment" { + scope = module.openai.id + role_definition_name = "Cognitive Services User" + principal_id = azurerm_user_assigned_identity.aks_workload_identity.principal_id + skip_service_principal_aad_check = true } -module "log_analytics_workspace" { - source = "./modules/log_analytics" - name = "${var.name_prefix}${var.log_analytics_workspace_name}" - location = var.location +resource "azurerm_federated_identity_credential" "federated_identity_credential" { + name = "${title(var.namespace)}FederatedIdentity" resource_group_name = azurerm_resource_group.rg.name + audience = ["api://AzureADTokenExchange"] + issuer = module.aks_cluster.oidc_issuer_url + parent_id = azurerm_user_assigned_identity.aks_workload_identity.id + subject = "system:serviceaccount:${var.namespace}:${var.service_account_name}" +} - solution_plan_map = { - ContainerInsights = { - product = "OMSGallery/ContainerInsights" - publisher = "Microsoft" - } - } +resource "azurerm_role_assignment" "network_contributor_assignment" { + scope = azurerm_resource_group.rg.id + role_definition_name = "Network Contributor" + principal_id = module.aks_cluster.aks_identity_principal_id + skip_service_principal_aad_check = true +} + +resource "azurerm_role_assignment" "acr_pull_assignment" { + role_definition_name = "AcrPull" + scope = module.container_registry.id + principal_id = module.aks_cluster.kubelet_identity_object_id + skip_service_principal_aad_check = true } From 0ee758d055dbe0ca8a7bfe74e6f35a1f65427493 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 15:16:13 -0800 Subject: [PATCH 032/119] Clean up --- .../terraform/modules/key_vault/main.tf | 7 -- .../terraform/modules/log_analytics/main.tf | 7 -- .../terraform/modules/nat_gateway/main.tf | 14 --- .../terraform/modules/node_pool/main.tf | 25 ---- .../terraform/modules/node_pool/outputs.tf | 4 - .../terraform/modules/node_pool/variables.tf | 119 ------------------ 6 files changed, 176 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/node_pool/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf index 02cce3be0..312190d28 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf @@ -4,7 +4,6 @@ resource "azurerm_key_vault" "key_vault" { resource_group_name = var.resource_group_name tenant_id = var.tenant_id sku_name = var.sku_name - tags = var.tags enabled_for_deployment = var.enabled_for_deployment enabled_for_disk_encryption = var.enabled_for_disk_encryption enabled_for_template_deployment = var.enabled_for_template_deployment @@ -22,12 +21,6 @@ resource "azurerm_key_vault" "key_vault" { ip_rules = var.ip_rules virtual_network_subnet_ids = var.virtual_network_subnet_ids } - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_monitor_diagnostic_setting" "settings" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf index fc3a1d85a..7e802cfe8 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf @@ -3,14 +3,7 @@ resource "azurerm_log_analytics_workspace" "log_analytics_workspace" { location = var.location resource_group_name = var.resource_group_name sku = var.sku - tags = var.tags retention_in_days = var.retention_in_days != "" ? var.retention_in_days : null - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_log_analytics_solution" "la_solution" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf index bb5d7c5b0..97b8f742e 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf @@ -5,13 +5,6 @@ resource "azurerm_public_ip" "nat_gategay_public_ip" { allocation_method = "Static" sku = "Standard" zones = var.zones - tags = var.tags - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_nat_gateway" "nat_gateway" { @@ -21,13 +14,6 @@ resource "azurerm_nat_gateway" "nat_gateway" { sku_name = var.sku_name idle_timeout_in_minutes = var.idle_timeout_in_minutes zones = var.zones - tags = var.tags - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_association" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf deleted file mode 100644 index a28e1582e..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/main.tf +++ /dev/null @@ -1,25 +0,0 @@ -resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { - kubernetes_cluster_id = var.kubernetes_cluster_id - name = var.name - vm_size = var.vm_size - mode = var.mode - node_labels = var.node_labels - node_taints = var.node_taints - zones = var.availability_zones - vnet_subnet_id = var.vnet_subnet_id - pod_subnet_id = var.pod_subnet_id - proximity_placement_group_id = var.proximity_placement_group_id - orchestrator_version = var.orchestrator_version - max_pods = var.max_pods - os_disk_size_gb = var.os_disk_size_gb - os_disk_type = var.os_disk_type - os_type = var.os_type - priority = var.priority - tags = var.tags - - lifecycle { - ignore_changes = [ - tags - ] - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/outputs.tf deleted file mode 100644 index 936f87b5c..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "id" { - description = "Specifies the resource id of the node pool" - value = azurerm_kubernetes_cluster_node_pool.node_pool.id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf deleted file mode 100644 index 2e2825bd6..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/node_pool/variables.tf +++ /dev/null @@ -1,119 +0,0 @@ -variable "name" { - description = "(Required) Specifies the name of the node pool." - type = string -} - -variable "kubernetes_cluster_id" { - description = "(Required) Specifies the resource id of the AKS cluster." - type = string -} - -variable "vm_size" { - description = "(Required) The SKU which should be used for the Virtual Machines used in this Node Pool. Changing this forces a new resource to be created." - type = string -} - -variable "availability_zones" { - description = "(Optional) A list of Availability Zones where the Nodes in this Node Pool should be created in. Changing this forces a new resource to be created." - type = list(string) - default = ["1", "2", "3"] -} - -variable "enable_host_encryption" { - description = "(Optional) Should the nodes in this Node Pool have host encryption enabled? Defaults to false." - type = bool - default = false -} - -variable "enable_node_public_ip" { - description = "(Optional) Should each node have a Public IP Address? Defaults to false. Changing this forces a new resource to be created." - type = bool - default = false -} - -variable "max_pods" { - description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." - type = number - default = 250 -} - -variable "mode" { - description = "(Optional) Should this Node Pool be used for System or User resources? Possible values are System and User. Defaults to User." - type = string - default = "User" -} - -variable "node_labels" { - description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. Changing this forces a new resource to be created." - type = map(any) - default = {} -} - -variable "node_taints" { - description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." - type = list(string) - default = [] -} - -variable "tags" { - description = "(Optional) Specifies the tags of the network security group" - default = {} -} - -variable "orchestrator_version" { - description = "(Required) Version of Kubernetes used for the Agents. If not specified, the latest recommended version will be used at provisioning time (but won't auto-upgrade)" - type = string -} - -variable "os_disk_size_gb" { - description = "(Optional) The Agent Operating System disk size in GB. Changing this forces a new resource to be created." - type = number - default = null -} - -variable "os_disk_type" { - description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." - type = string - default = "Ephemeral" -} - -variable "os_type" { - description = "(Optional) The Operating System which should be used for this Node Pool. Changing this forces a new resource to be created. Possible values are Linux and Windows. Defaults to Linux." - type = string - default = "Linux" -} - -variable "priority" { - description = "(Optional) The Priority for Virtual Machines within the Virtual Machine Scale Set that powers this Node Pool. Possible values are Regular and Spot. Defaults to Regular. Changing this forces a new resource to be created." - type = string - default = "Regular" -} - -variable "proximity_placement_group_id" { - description = "(Optional) The ID of the Proximity Placement Group where the Virtual Machine Scale Set that powers this Node Pool will be placed. Changing this forces a new resource to be created." - type = string - default = null -} - -variable "vnet_subnet_id" { - description = "(Optional) The ID of the Subnet where this Node Pool should exist." - type = string - default = null -} - -variable "pod_subnet_id" { - description = "(Optional) The ID of the Subnet where the pods in the system node pool should exist. Changing this forces a new resource to be created." - type = string - default = null -} - -variable "resource_group_name" { - description = "Specifies the resource group name" - type = string -} - -variable "oidc_issuer_enabled" { - description = " (Optional) Enable or Disable the OIDC issuer URL." - type = bool - default = true -} From 606620d85809aab60320026cdb8641f6d3403ab6 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 15:38:56 -0800 Subject: [PATCH 033/119] Dead code --- .../AksOpenAiTerraform/terraform/main.tf | 44 ++-- .../modules/diagnostic_setting/outputs.tf | 9 - .../terraform/modules/firewall/main.tf | 248 ------------------ .../terraform/modules/firewall/outputs.tf | 4 - .../terraform/modules/firewall/variables.tf | 80 ------ .../terraform/modules/nat_gateway/main.tf | 12 +- .../terraform/modules/nat_gateway/output.tf | 14 - .../modules/nat_gateway/variables.tf | 40 +-- .../modules/network_security_group/main.tf | 41 --- .../modules/network_security_group/outputs.tf | 4 - .../network_security_group/variables.tf | 36 --- .../terraform/modules/route_table/main.tf | 22 -- .../modules/route_table/variables.tf | 35 --- 13 files changed, 36 insertions(+), 553 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/variables.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/route_table/variables.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index f08c2037d..26458b859 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -30,7 +30,7 @@ resource "random_string" "storage_account_suffix" { } resource "azurerm_resource_group" "rg" { - name = "${var.name_prefix}${var.resource_group_name}" + name = "${var.name_prefix}-rg" location = var.location } @@ -97,16 +97,18 @@ module "storage_account" { name = "boot${random_string.storage_account_suffix.result}" location = var.location resource_group_name = azurerm_resource_group.rg.name - account_kind = "StorageV2" - account_tier = "Standard" - replication_type = "LRS" + + account_kind = "StorageV2" + account_tier = "Standard" + replication_type = "LRS" } module "key_vault" { - source = "./modules/key_vault" - name = "${var.name_prefix}KeyVault" - location = var.location - resource_group_name = azurerm_resource_group.rg.name + source = "./modules/key_vault" + name = "${var.name_prefix}KeyVault" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + tenant_id = data.azurerm_client_config.current.tenant_id sku_name = "standard" enabled_for_deployment = true @@ -122,10 +124,11 @@ module "key_vault" { } module "deployment_script" { - source = "./modules/deployment_script" - name = "${var.name_prefix}BashScript" - location = var.location - resource_group_name = azurerm_resource_group.rg.name + source = "./modules/deployment_script" + name = "${var.name_prefix}BashScript" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + azure_cli_version = "2.9.1" managed_identity_name = "${var.name_prefix}ScriptManagedIdentity" aks_cluster_name = module.aks_cluster.name @@ -220,18 +223,17 @@ module "nat_gateway" { location = var.location resource_group_name = azurerm_resource_group.rg.name - sku_name = "Standard" - idle_timeout_in_minutes = 4 - zones = ["1"] - subnet_ids = module.virtual_network.subnet_ids + subnet_ids = module.virtual_network.subnet_ids } module "bastion_host" { - source = "./modules/bastion_host" - name = "${var.name_prefix}BastionHost" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - subnet_id = module.virtual_network.subnet_ids["AzureBastionSubnet"] + source = "./modules/bastion_host" + name = "${var.name_prefix}BastionHost" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + + subnet_id = module.virtual_network.subnet_ids["AzureBastionSubnet"] + log_analytics_workspace_id = module.log_analytics_workspace.id log_analytics_retention_days = var.log_analytics_retention_days } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf deleted file mode 100644 index 3d727607e..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/outputs.tf +++ /dev/null @@ -1,9 +0,0 @@ -output "name" { - value = azurerm_key_vault.key_vault.name - description = "Specifies the name of the key vault." -} - -output "id" { - value = azurerm_key_vault.key_vault.id - description = "Specifies the resource id of the key vault." -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf deleted file mode 100644 index 3ce12243d..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/main.tf +++ /dev/null @@ -1,248 +0,0 @@ -resource "azurerm_public_ip" "pip" { - name = var.pip_name - resource_group_name = var.resource_group_name - location = var.location - zones = var.zones - allocation_method = "Static" - sku = "Standard" -} - -resource "azurerm_firewall" "firewall" { - name = var.name - resource_group_name = var.resource_group_name - location = var.location - zones = var.zones - threat_intel_mode = var.threat_intel_mode - sku_name = var.sku_name - sku_tier = var.sku_tier - firewall_policy_id = azurerm_firewall_policy.policy.id - - ip_configuration { - name = "fw_ip_config" - subnet_id = var.subnet_id - public_ip_address_id = azurerm_public_ip.pip.id - } -} - -resource "azurerm_firewall_policy" "policy" { - name = "${var.name}Policy" - resource_group_name = var.resource_group_name - location = var.location -} - -resource "azurerm_firewall_policy_rule_collection_group" "policy" { - name = "AksEgressPolicyRuleCollectionGroup" - firewall_policy_id = azurerm_firewall_policy.policy.id - priority = 500 - - application_rule_collection { - name = "ApplicationRules" - priority = 500 - action = "Allow" - - rule { - name = "AllowMicrosoftFqdns" - source_addresses = ["*"] - - destination_fqdns = [ - "*.cdn.mscr.io", - "mcr.microsoft.com", - "*.data.mcr.microsoft.com", - "management.azure.com", - "login.microsoftonline.com", - "acs-mirror.azureedge.net", - "dc.services.visualstudio.com", - "*.opinsights.azure.com", - "*.oms.opinsights.azure.com", - "*.microsoftonline.com", - "*.monitoring.azure.com", - ] - - protocols { - port = "80" - type = "Http" - } - - protocols { - port = "443" - type = "Https" - } - } - - rule { - name = "AllowFqdnsForOsUpdates" - source_addresses = ["*"] - - destination_fqdns = [ - "download.opensuse.org", - "security.ubuntu.com", - "ntp.ubuntu.com", - "packages.microsoft.com", - "snapcraft.io" - ] - - protocols { - port = "80" - type = "Http" - } - - protocols { - port = "443" - type = "Https" - } - } - - rule { - name = "AllowImagesFqdns" - source_addresses = ["*"] - - destination_fqdns = [ - "auth.docker.io", - "registry-1.docker.io", - "production.cloudflare.docker.com" - ] - - protocols { - port = "80" - type = "Http" - } - - protocols { - port = "443" - type = "Https" - } - } - - rule { - name = "AllowBing" - source_addresses = ["*"] - - destination_fqdns = [ - "*.bing.com" - ] - - protocols { - port = "80" - type = "Http" - } - - protocols { - port = "443" - type = "Https" - } - } - - rule { - name = "AllowGoogle" - source_addresses = ["*"] - - destination_fqdns = [ - "*.google.com" - ] - - protocols { - port = "80" - type = "Http" - } - - protocols { - port = "443" - type = "Https" - } - } - } - - network_rule_collection { - name = "NetworkRules" - priority = 400 - action = "Allow" - - rule { - name = "Time" - source_addresses = ["*"] - destination_ports = ["123"] - destination_addresses = ["*"] - protocols = ["UDP"] - } - - rule { - name = "DNS" - source_addresses = ["*"] - destination_ports = ["53"] - destination_addresses = ["*"] - protocols = ["UDP"] - } - - rule { - name = "ServiceTags" - source_addresses = ["*"] - destination_ports = ["*"] - destination_addresses = [ - "AzureContainerRegistry", - "MicrosoftContainerRegistry", - "AzureActiveDirectory" - ] - protocols = ["Any"] - } - - rule { - name = "Internet" - source_addresses = ["*"] - destination_ports = ["*"] - destination_addresses = ["*"] - protocols = ["TCP"] - } - } - - lifecycle { - ignore_changes = [ - application_rule_collection, - network_rule_collection, - nat_rule_collection - ] - } -} - -resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "FirewallDiagnosticsSettings" - target_resource_id = azurerm_firewall.firewall.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - enabled_log { - category = "AzureFirewallApplicationRule" - } - - enabled_log { - category = "AzureFirewallNetworkRule" - } - - enabled_log { - category = "AzureFirewallDnsProxy" - } - - metric { - category = "AllMetrics" - } -} - -resource "azurerm_monitor_diagnostic_setting" "pip_settings" { - name = "FirewallDdosDiagnosticsSettings" - target_resource_id = azurerm_public_ip.pip.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - enabled_log { - category = "DDoSProtectionNotifications" - } - - enabled_log { - category = "DDoSMitigationFlowLogs" - } - - enabled_log { - category = "DDoSMitigationReports" - } - - metric { - category = "AllMetrics" - } -} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf deleted file mode 100644 index f280bb2c1..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "private_ip_address" { - description = "Specifies the private IP address of the firewall." - value = azurerm_firewall.firewall.ip_configuration[0].private_ip_address -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf deleted file mode 100644 index aa67baa3b..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/firewall/variables.tf +++ /dev/null @@ -1,80 +0,0 @@ -variable "name" { - description = "Specifies the firewall name" - type = string -} - -variable "sku_name" { - description = "(Required) SKU name of the Firewall. Possible values are AZFW_Hub and AZFW_VNet. Changing this forces a new resource to be created." - default = "AZFW_VNet" - type = string - - validation { - condition = contains(["AZFW_Hub", "AZFW_VNet"], var.sku_name) - error_message = "The value of the sku name property of the firewall is invalid." - } -} - -variable "sku_tier" { - description = "(Required) SKU tier of the Firewall. Possible values are Premium, Standard, and Basic." - default = "Standard" - type = string - - validation { - condition = contains(["Premium", "Standard", "Basic"], var.sku_tier) - error_message = "The value of the sku tier property of the firewall is invalid." - } -} - -variable "resource_group_name" { - description = "Specifies the resource group name" - type = string -} - -variable "location" { - description = "Specifies the location where firewall will be deployed" - type = string -} - -variable "threat_intel_mode" { - description = "(Optional) The operation mode for threat intelligence-based filtering. Possible values are: Off, Alert, Deny. Defaults to Alert." - default = "Alert" - type = string - - validation { - condition = contains(["Off", "Alert", "Deny"], var.threat_intel_mode) - error_message = "The threat intel mode is invalid." - } -} - -variable "zones" { - description = "Specifies the availability zones of the Azure Firewall" - default = ["1", "2", "3"] - type = list(string) -} - -variable "pip_name" { - description = "Specifies the firewall public IP name" - type = string - default = "azure-fw-ip" -} - -variable "subnet_id" { - description = "Subnet ID" - type = string -} - -variable "tags" { - description = "(Optional) Specifies the tags of the storage account" - default = {} -} - -variable "log_analytics_workspace_id" { - description = "Specifies the log analytics workspace id" - type = string -} - -variable "log_analytics_retention_days" { - description = "Specifies the number of days of the retention policy" - type = number - default = 7 -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf index 97b8f742e..dc8da73a6 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf @@ -1,19 +1,21 @@ +locals { + zones = ["1"] +} + resource "azurerm_public_ip" "nat_gategay_public_ip" { name = "${var.name}PublicIp" location = var.location resource_group_name = var.resource_group_name allocation_method = "Static" - sku = "Standard" - zones = var.zones + zones = local.zones } resource "azurerm_nat_gateway" "nat_gateway" { name = var.name location = var.location resource_group_name = var.resource_group_name - sku_name = var.sku_name - idle_timeout_in_minutes = var.idle_timeout_in_minutes - zones = var.zones + idle_timeout_in_minutes = 4 + zones = local.zones } resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_association" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf deleted file mode 100644 index 2b9ce3bb5..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/output.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "name" { - value = azurerm_nat_gateway.nat_gateway.name - description = "Specifies the name of the Azure NAT Gateway" -} - -output "id" { - value = azurerm_nat_gateway.nat_gateway.id - description = "Specifies the resource id of the Azure NAT Gateway" -} - -output "public_ip_address" { - value = azurerm_public_ip.nat_gategay_public_ip.ip_address - description = "Contains the public IP address of the Azure NAT Gateway." -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf index 14f745663..0accf9ced 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf @@ -1,43 +1,15 @@ -variable "resource_group_name" { - description = "(Required) Specifies the resource group name" - type = string -} - -variable "location" { - description = "(Required) Specifies the location of the Azure OpenAI Service" - type = string -} - variable "name" { - description = "(Required) Specifies the name of the Azure OpenAI Service" - type = string + type = string } -variable "tags" { - description = "(Optional) Specifies the tags of the Azure OpenAI Service" - type = map(any) - default = {} -} - -variable "sku_name" { - description = "(Optional) The SKU which should be used. At this time the only supported value is Standard. Defaults to Standard" - type = string - default = "Standard" -} - -variable "idle_timeout_in_minutes" { - description = "(Optional) The idle timeout which should be used in minutes. Defaults to 4." - type = number - default = 4 +variable "location" { + type = string } -variable "zones" { - description = " (Optional) A list of Availability Zones in which this NAT Gateway should be located. Changing this forces a new NAT Gateway to be created." - type = list(string) - default = [] +variable "resource_group_name" { + type = string } variable "subnet_ids" { - description = "(Required) A map of subnet ids to associate with the NAT Gateway" - type = map(string) + type = map(string) } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf deleted file mode 100644 index be9f9cbf2..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/main.tf +++ /dev/null @@ -1,41 +0,0 @@ -resource "azurerm_network_security_group" "nsg" { - name = var.name - resource_group_name = var.resource_group_name - location = var.location - tags = var.tags - - dynamic "security_rule" { - for_each = try(var.security_rules, []) - content { - name = try(security_rule.value.name, null) - priority = try(security_rule.value.priority, null) - direction = try(security_rule.value.direction, null) - access = try(security_rule.value.access, null) - protocol = try(security_rule.value.protocol, null) - source_port_range = try(security_rule.value.source_port_range, null) - source_port_ranges = try(security_rule.value.source_port_ranges, null) - destination_port_range = try(security_rule.value.destination_port_range, null) - destination_port_ranges = try(security_rule.value.destination_port_ranges, null) - source_address_prefix = try(security_rule.value.source_address_prefix, null) - source_address_prefixes = try(security_rule.value.source_address_prefixes, null) - destination_address_prefix = try(security_rule.value.destination_address_prefix, null) - destination_address_prefixes = try(security_rule.value.destination_address_prefixes, null) - source_application_security_group_ids = try(security_rule.value.source_application_security_group_ids, null) - destination_application_security_group_ids = try(security_rule.value.destination_application_security_group_ids, null) - } - } -} - -resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "NetworkSecurityDiagnosticsSettings" - target_resource_id = azurerm_network_security_group.nsg.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - enabled_log { - category = "NetworkSecurityGroupEvent" - } - - enabled_log { - category = "NetworkSecurityGroupRuleCounter" - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/outputs.tf deleted file mode 100644 index ca2a13e32..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "id" { - description = "Specifies the resource id of the network security group" - value = azurerm_network_security_group.nsg.id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/variables.tf deleted file mode 100644 index 1de3c61ad..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/network_security_group/variables.tf +++ /dev/null @@ -1,36 +0,0 @@ -variable "name" { - description = "(Required) Specifies the name of the network security group" - type = string -} - -variable "resource_group_name" { - description = "(Required) Specifies the resource group name of the network security group" - type = string -} - -variable "location" { - description = "(Required) Specifies the location of the network security group" - type = string -} - -variable "security_rules" { - description = "(Optional) Specifies the security rules of the network security group" - type = list(object) - default = [] -} - -variable "tags" { - description = "(Optional) Specifies the tags of the network security group" - default = {} -} - -variable "log_analytics_workspace_id" { - description = "Specifies the log analytics workspace resource id" - type = string -} - -variable "log_analytics_retention_days" { - description = "Specifies the number of days of the retention policy" - type = number - default = 7 -} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf deleted file mode 100644 index 58971058f..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/route_table/main.tf +++ /dev/null @@ -1,22 +0,0 @@ -data "azurerm_client_config" "current" { -} - -resource "azurerm_route_table" "rt" { - name = var.route_table_name - location = var.location - resource_group_name = var.resource_group_name - - route { - name = "kubenetfw_fw_r" - address_prefix = "0.0.0.0/0" - next_hop_type = "VirtualAppliance" - next_hop_in_ip_address = var.firewall_private_ip - } -} - -resource "azurerm_subnet_route_table_association" "subnet_association" { - for_each = var.subnets_to_associate - - subnet_id = "/subscriptions/${each.value.subscription_id}/resourceGroups/${each.value.resource_group_name}/providers/Microsoft.Network/virtualNetworks/${each.value.virtual_network_name}/subnets/${each.key}" - route_table_id = azurerm_route_table.rt.id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/route_table/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/route_table/variables.tf deleted file mode 100644 index 6102e8065..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/route_table/variables.tf +++ /dev/null @@ -1,35 +0,0 @@ -variable "resource_group_name" { - description = "Resource group where RouteTable will be deployed" - type = string -} - -variable "location" { - description = "Location where RouteTable will be deployed" - type = string -} - -variable "route_table_name" { - description = "RouteTable name" - type = string -} - -variable "route_name" { - description = "AKS route name" - type = string -} - -variable "firewall_private_ip" { - description = "Firewall private IP" - type = string -} - -variable "subnets_to_associate" { - description = "(Optional) Specifies the subscription id, resource group name, and name of the subnets to associate" - type = map(any) - default = {} -} - -variable "tags" { - description = "(Optional) Specifies the tags of the storage account" - default = {} -} \ No newline at end of file From fb9d8de1413ea124165b5c87da82d5f4578c5fd3 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 15:46:44 -0800 Subject: [PATCH 034/119] More inline --- .../terraform/.terraform.lock.hcl | 57 +++++++++---------- .../AksOpenAiTerraform/terraform/main.tf | 51 ++++++++++------- .../AksOpenAiTerraform/terraform/variables.tf | 49 +--------------- 3 files changed, 61 insertions(+), 96 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl index 9df9eb753..6b63a37e1 100644 --- a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl +++ b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl @@ -2,42 +2,41 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/azure/azapi" { - version = "2.0.1" - constraints = "~> 2.0.1" + version = "2.2.0" hashes = [ - "h1:VJpm9+TaZ4SC6ncXCiiE+jWmLKZRbrd4KOt79iMIicU=", - "zh:3df16ed604be5f4ccd5d52a02c2681d8eb2f5a4462625c983cb17c20cdf0bfb2", - "zh:4efd9961ea52990e21385086f0b3324edfb534ea6a8f0f6ba146a74bfb56aa63", - "zh:5561418efc9744c9873855a146226608778e29b4c0c3b3872634ef2da2d86593", - "zh:7ebcb4c6ca71c87850df67d4e5f79ce4a036d4131b8c11ae0b9b8787353843b8", - "zh:81a9259cb1e45507e9431794fbd354dd4d8b78c6a9508b0bfa108b00e6ad23cb", - "zh:8c1836fa186272347f97c7a3884556979618d1b93721e8a24203d90ff4efbd40", - "zh:a72bdd43a11a383525764720d24cb78ec5d9f1167f129d05448108fef1ba7af3", - "zh:ade9d17c6b8717e7b04af5a9d1a948d047ac4dcf6affb2485afa3ad0a2eaee15", - "zh:b3c5bfcab98251cb0c157dbe78dc6d0864c9bf364d316003c84c1e624a3c3524", - "zh:c33b872a2473a9b052add89e4557d361b0ebaa42865e99b95465050d2c858d43", - "zh:efe425f8ecd4d79448214c93ef10881b3b74cf2d9b5211d76f05aced22621eb4", - "zh:ff704c5e73e832507367d9d962b6b53c0ca3c724689f0974feffd5339c3db18a", + "h1:ng+uFmo5IvLRJEVU/sEN81JO9HB32WOtKQT4rM7L/Ic=", + "zh:062be5d8272cac297a88c2057449f449ea6906c4121ba3dfdeb5cecb3ff91178", + "zh:1fd9abec3ffcbf8d0244408334e9bfc8f49ada50978cd73ee0ed5f8560987267", + "zh:48e84b0302af99d7e7f4248a724088fb1c34aeee78c9ca63ec5a9464ec5054a0", + "zh:4e7302883fd9dd83bfbbcd72ebd55f83d8b16ccc6d12d1573d578058e604d5cf", + "zh:5b6e181e32cbf62f5d2ce34f9d6d9ffe17192e24943450bbe335e1baf0494e66", + "zh:62d525d426c6d5f10109ab04a9abc231b204ea413238f5690f69b420a8b8583a", + "zh:90aab23497ec9c7af44ad9ea1a1d6063dc3331334915e1c549527a73c2c6948d", + "zh:91ecf30a01df5e832191e0c55c87f8403a1f584796fd70f9c9c913d35c2e2a37", + "zh:bc3a5db5e4b9695a69dff47cf1e7184eaf5564d3dc50f231cbcbf535dd140d19", + "zh:cb566bec2676511bf4722e24d0dfc9bf58aff78af38b8e0864970f20d263118f", + "zh:d4fa0c1462b389cee313e1c152e00f5dfc175a1be3615d3b23b526a8581e39a5", + "zh:f8136b0f41045a1e5a6dedc6b6fb055faee3d825f84a3192312e3ac5d057ff72", ] } provider "registry.terraform.io/hashicorp/azurerm" { - version = "4.11.0" - constraints = "4.11.0" + version = "4.16.0" + constraints = "~> 4.16.0" hashes = [ - "h1:l1igOrMmeHJHXEj9eLkx9Uiq/iKKbukoRuPUIDGBY/8=", - "zh:026808a5ff8bce161518d503bfc57c4a95637d67e923a94382c8e878c96aaf00", - "zh:13473ebb56ed701fdd8c288a220cef3ec6ee170fb1ac45c6ce5a612848e64690", - "zh:36667374d31509456fd928f651fc1ccc7438c53bc99cf9ec3b6ec6e7f791394e", - "zh:5f44e16aab36a93391ce81b9a93b694fecf11f71615f2414ee40bb5e211d3dbb", - "zh:9310e860f9236d0f7171e05444ca85e239f0938b9fb08ec3bfd9712a14013308", - "zh:aaf6ea1f68526a175e84424710b06dd6cf8987b404206cc581692560c1530810", - "zh:b6d1965af0aed85f3eccaaec5dae90f59632bf07e2bf5b7473359a7c761872a5", - "zh:c642675ea2d8e1f1bb440016238ab25fa4270cb155b01e90598161488df47128", - "zh:d22d07834c2a5da6ce7054699d4f708277fccb63436cfbf6c90c58cddddba408", - "zh:eceb91d652ea9145531129c7da50603e9415812f639acbf1720d51f878798fb8", - "zh:f26bf55ce68c1ed6e316ee70652bc3cc357987ea4b3caf6f835405850c6897e0", + "h1:UNZga7kYMfYfDHmuP6LvHmJNXlb3fyvRY1tA9ol6yY4=", + "zh:2035e461a94bd4180557a06f8e56f228a8a035608d0dac4d08e5870cf9265276", + "zh:3f15778a22ef1b9d0fa28670e5ea6ef1094b0be2533f43f350a2ef15d471b353", + "zh:4f1a4d03b008dd958bcd6bf82cf088fbaa9c121be2fd35e10e6b06c6e8f6aaa1", + "zh:5859f31c342364e849b4f8c437a46f33e927fa820244d0732b8d2ec74a95712d", + "zh:693d0f15512ca8c6b5e999b3a7551503feb06b408b3836bc6a6403e518b9ddab", + "zh:7f4912bec5b04f5156935292377c12484c13582151eb3c2555df409a7e5fb6e0", + "zh:bb9a509497f3a131c52fac32348919bf1b9e06c69a65f24607b03f7b56fb47b6", + "zh:c1b0c64e49ac591fd038ad71e71403ff71c07476e27e8da718c29f0028ea6d0d", + "zh:dd4ca432ee14eb0bb0cdc0bb463c8675b8ef02497be870a20d8dfee3e7fe52b3", + "zh:df58bb7fea984d2b11709567842ca4d55b3f24e187aa6be99e3677f55cbbe7da", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f7fb37704da50c096f9c7c25e8a95fe73ce1d3c5aab0d616d506f07bc5cfcdd8", ] } diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 26458b859..bd2891602 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -14,6 +14,19 @@ provider "azurerm" { data "azurerm_client_config" "current" { } +locals { + log_analytics_workspace_name = "Workspace" + log_analytics_retention_days = 30 + + system_node_pool_subnet_name = "SystemSubnet" + user_node_pool_subnet_name = "UserSubnet" + pod_subnet_name = "PodSubnet" + vm_subnet_name = "VmSubnet" + + namespace = "magic8ball" + service_account_name = "magic8ball-sa" +} + resource "random_string" "prefix" { length = 6 special = false @@ -56,7 +69,7 @@ module "openai" { custom_subdomain_name = lower("${var.name_prefix}OpenAi") public_network_access_enabled = true log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = var.log_analytics_retention_days + log_analytics_retention_days = local.log_analytics_retention_days } module "aks_cluster" { @@ -68,9 +81,9 @@ module "aks_cluster" { kubernetes_version = "1.32" sku_tier = "Free" - user_node_pool_subnet_name = var.user_node_pool_subnet_name - system_node_pool_subnet_name = var.system_node_pool_subnet_name - pod_subnet_name = var.pod_subnet_name + user_node_pool_subnet_name = local.user_node_pool_subnet_name + system_node_pool_subnet_name = local.system_node_pool_subnet_name + pod_subnet_name = local.pod_subnet_name log_analytics_workspace_id = module.log_analytics_workspace.id @@ -120,7 +133,7 @@ module "key_vault" { bypass = "AzureServices" default_action = "Allow" log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = var.log_analytics_retention_days + log_analytics_retention_days = local.log_analytics_retention_days } module "deployment_script" { @@ -133,8 +146,8 @@ module "deployment_script" { managed_identity_name = "${var.name_prefix}ScriptManagedIdentity" aks_cluster_name = module.aks_cluster.name hostname = "magic8ball.contoso.com" - namespace = var.namespace - service_account_name = var.service_account_name + namespace = local.namespace + service_account_name = local.service_account_name email = var.email primary_script_uri = "https://paolosalvatori.blob.core.windows.net/scripts/install-nginx-via-helm-and-create-sa.sh" tenant_id = data.azurerm_client_config.current.tenant_id @@ -148,7 +161,7 @@ module "deployment_script" { module "log_analytics_workspace" { source = "./modules/log_analytics" - name = "${var.name_prefix}${var.log_analytics_workspace_name}" + name = "${var.name_prefix}${local.log_analytics_workspace_name}" location = var.location resource_group_name = azurerm_resource_group.rg.name @@ -174,21 +187,21 @@ module "virtual_network" { address_space = ["10.0.0.0/8"] subnets = [ { - name : var.system_node_pool_subnet_name + name : local.system_node_pool_subnet_name address_prefixes : ["10.240.0.0/16"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false delegation : null }, { - name : var.user_node_pool_subnet_name + name : local.user_node_pool_subnet_name address_prefixes : ["10.241.0.0/16"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false delegation : null }, { - name : var.pod_subnet_name + name : local.pod_subnet_name address_prefixes : ["10.242.0.0/16"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false @@ -201,7 +214,7 @@ module "virtual_network" { } }, { - name : var.vm_subnet_name + name : local.vm_subnet_name address_prefixes : ["10.243.1.0/24"] private_endpoint_network_policies : "Enabled" private_link_service_network_policies_enabled : false @@ -235,7 +248,7 @@ module "bastion_host" { subnet_id = module.virtual_network.subnet_ids["AzureBastionSubnet"] log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = var.log_analytics_retention_days + log_analytics_retention_days = local.log_analytics_retention_days } ############################################################################### @@ -297,7 +310,7 @@ module "openai_private_endpoint" { name = "${module.openai.name}PrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name - subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.openai.id is_manual_connection = false subresource_name = "account" @@ -310,7 +323,7 @@ module "acr_private_endpoint" { name = "${module.container_registry.name}PrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name - subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.container_registry.id is_manual_connection = false subresource_name = "registry" @@ -323,7 +336,7 @@ module "key_vault_private_endpoint" { name = "${module.key_vault.name}PrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name - subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.key_vault.id is_manual_connection = false subresource_name = "vault" @@ -336,7 +349,7 @@ module "blob_private_endpoint" { name = "${var.name_prefix}BlobStoragePrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name - subnet_id = module.virtual_network.subnet_ids[var.vm_subnet_name] + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.storage_account.id is_manual_connection = false subresource_name = "blob" @@ -361,12 +374,12 @@ resource "azurerm_role_assignment" "cognitive_services_user_assignment" { } resource "azurerm_federated_identity_credential" "federated_identity_credential" { - name = "${title(var.namespace)}FederatedIdentity" + name = "${title(local.namespace)}FederatedIdentity" resource_group_name = azurerm_resource_group.rg.name audience = ["api://AzureADTokenExchange"] issuer = module.aks_cluster.oidc_issuer_url parent_id = azurerm_user_assigned_identity.aks_workload_identity.id - subject = "system:serviceaccount:${var.namespace}:${var.service_account_name}" + subject = "system:serviceaccount:${local.namespace}:${local.service_account_name}" } resource "azurerm_role_assignment" "network_contributor_assignment" { diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index aeb7ff5aa..a5f4e45ef 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -1,15 +1,6 @@ variable "name_prefix" { type = string -} - -variable "log_analytics_workspace_name" { - default = "Workspace" - type = string -} - -variable "log_analytics_retention_days" { - type = number - default = 30 + default = "AksOpenAiTerraform" } variable "location" { @@ -17,45 +8,7 @@ variable "location" { type = string } -variable "resource_group_name" { - default = "RG" - type = string -} - -variable "system_node_pool_subnet_name" { - default = "SystemSubnet" - type = string -} - -variable "user_node_pool_subnet_name" { - default = "UserSubnet" - type = string -} - -variable "pod_subnet_name" { - default = "PodSubnet" - type = string -} - -variable "vm_subnet_name" { - default = "VmSubnet" - type = string -} - -variable "namespace" { - description = "Specifies the namespace of the workload application that accesses the Azure OpenAI Service." - type = string - default = "magic8ball" -} - -variable "service_account_name" { - description = "Specifies the name of the service account of the workload application that accesses the Azure OpenAI Service." - type = string - default = "magic8ball-sa" -} - variable "email" { - description = "Specifies the email address for the cert-manager cluster issuer." type = string default = "paolos@microsoft.com" } \ No newline at end of file From 063c89dd127a17d30962d863774998b3a97ffc7e Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 16:42:29 -0800 Subject: [PATCH 035/119] Fixes --- .../AksOpenAiTerraform/terraform/main.tf | 135 ++++++++---------- .../terraform/modules/aks/main.tf | 8 +- .../terraform/modules/aks/variables.tf | 4 + .../modules/container_registry/outputs.tf | 7 + .../modules/storage_account/outputs.tf | 3 + .../terraform/modules/virtual_network/main.tf | 6 +- .../modules/virtual_network/outputs.tf | 7 + .../modules/virtual_network/variables.tf | 15 +- .../AksOpenAiTerraform/terraform/variables.tf | 8 +- 9 files changed, 99 insertions(+), 94 deletions(-) create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index bd2891602..19fc006ed 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -15,19 +15,53 @@ data "azurerm_client_config" "current" { } locals { - log_analytics_workspace_name = "Workspace" - log_analytics_retention_days = 30 + log_analytics_workspace_name = "Workspace" + log_analytics_retention_days = 30 - system_node_pool_subnet_name = "SystemSubnet" - user_node_pool_subnet_name = "UserSubnet" - pod_subnet_name = "PodSubnet" - vm_subnet_name = "VmSubnet" + system_node_pool_subnet_name = "SystemSubnet" + user_node_pool_subnet_name = "UserSubnet" + pod_subnet_name = "PodSubnet" + vm_subnet_name = "VmSubnet" - namespace = "magic8ball" - service_account_name = "magic8ball-sa" + namespace = "magic8ball" + service_account_name = "magic8ball-sa" + + subnets = [ + { + name : local.system_node_pool_subnet_name + address_prefixes : ["10.240.0.0/16"] + delegation = null + }, + { + name : local.user_node_pool_subnet_name + address_prefixes : ["10.241.0.0/16"] + delegation = null + }, + { + name : local.vm_subnet_name + address_prefixes : ["10.243.1.0/24"] + delegation = null + }, + { + name : "AzureBastionSubnet" + address_prefixes : ["10.243.2.0/24"] + delegation = null + }, + { + name : local.pod_subnet_name + address_prefixes : ["10.242.0.0/16"] + delegation = { + name = "delegation" + service_delegation = { + name = "Microsoft.ContainerService/managedClusters" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } + }, + ] } -resource "random_string" "prefix" { +resource "random_string" "rg_suffix" { length = 6 special = false upper = false @@ -43,7 +77,7 @@ resource "random_string" "storage_account_suffix" { } resource "azurerm_resource_group" "rg" { - name = "${var.name_prefix}-rg" + name = "${var.name_prefix}-${random_string.rg_suffix}-rg" location = var.location } @@ -55,6 +89,7 @@ module "openai" { name = "${var.name_prefix}OpenAi" location = var.location resource_group_name = azurerm_resource_group.rg.name + sku_name = "S0" deployments = [ { @@ -78,6 +113,7 @@ module "aks_cluster" { location = var.location resource_group_name = azurerm_resource_group.rg.name resource_group_id = azurerm_resource_group.rg.id + tenant_id = data.azurerm_client_config.current.tenant_id kubernetes_version = "1.32" sku_tier = "Free" @@ -178,56 +214,14 @@ module "log_analytics_workspace" { ############################################################################### module "virtual_network" { source = "./modules/virtual_network" - vnet_name = "AksVNet" + name = "AksVNet" location = var.location resource_group_name = azurerm_resource_group.rg.name log_analytics_workspace_id = module.log_analytics_workspace.id address_space = ["10.0.0.0/8"] - subnets = [ - { - name : local.system_node_pool_subnet_name - address_prefixes : ["10.240.0.0/16"] - private_endpoint_network_policies : "Enabled" - private_link_service_network_policies_enabled : false - delegation : null - }, - { - name : local.user_node_pool_subnet_name - address_prefixes : ["10.241.0.0/16"] - private_endpoint_network_policies : "Enabled" - private_link_service_network_policies_enabled : false - delegation : null - }, - { - name : local.pod_subnet_name - address_prefixes : ["10.242.0.0/16"] - private_endpoint_network_policies : "Enabled" - private_link_service_network_policies_enabled : false - delegation = { - name = "delegation" - service_delegation = { - name = "Microsoft.ContainerService/managedClusters" - actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] - } - } - }, - { - name : local.vm_subnet_name - address_prefixes : ["10.243.1.0/24"] - private_endpoint_network_policies : "Enabled" - private_link_service_network_policies_enabled : false - delegation : null - }, - { - name : "AzureBastionSubnet" - address_prefixes : ["10.243.2.0/24"] - private_endpoint_network_policies : "Enabled" - private_link_service_network_policies_enabled : false - delegation : null - } - ] + subnets = local.subnets } module "nat_gateway" { @@ -236,7 +230,7 @@ module "nat_gateway" { location = var.location resource_group_name = azurerm_resource_group.rg.name - subnet_ids = module.virtual_network.subnet_ids + subnet_ids = module.virtual_network.subnet_ids[local.system_node_pool_subnet_name] } module "bastion_host" { @@ -350,7 +344,7 @@ module "blob_private_endpoint" { location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] - private_connection_resource_id = module.storage_account.id + private_connection_resource_id = module.storage_account.name is_manual_connection = false subresource_name = "blob" private_dns_zone_group_name = "BlobPrivateDnsZoneGroup" @@ -358,7 +352,7 @@ module "blob_private_endpoint" { } ############################################################################### -# Identities +# Identities/Roles ############################################################################### resource "azurerm_user_assigned_identity" "aks_workload_identity" { name = "${var.name_prefix}WorkloadManagedIdentity" @@ -366,13 +360,6 @@ resource "azurerm_user_assigned_identity" "aks_workload_identity" { location = var.location } -resource "azurerm_role_assignment" "cognitive_services_user_assignment" { - scope = module.openai.id - role_definition_name = "Cognitive Services User" - principal_id = azurerm_user_assigned_identity.aks_workload_identity.principal_id - skip_service_principal_aad_check = true -} - resource "azurerm_federated_identity_credential" "federated_identity_credential" { name = "${title(local.namespace)}FederatedIdentity" resource_group_name = azurerm_resource_group.rg.name @@ -382,16 +369,20 @@ resource "azurerm_federated_identity_credential" "federated_identity_credential" subject = "system:serviceaccount:${local.namespace}:${local.service_account_name}" } +resource "azurerm_role_assignment" "cognitive_services_user_assignment" { + scope = module.openai.id + role_definition_name = "Cognitive Services User" + principal_id = azurerm_user_assigned_identity.aks_workload_identity.principal_id +} + resource "azurerm_role_assignment" "network_contributor_assignment" { - scope = azurerm_resource_group.rg.id - role_definition_name = "Network Contributor" - principal_id = module.aks_cluster.aks_identity_principal_id - skip_service_principal_aad_check = true + scope = azurerm_resource_group.rg.id + role_definition_name = "Network Contributor" + principal_id = module.aks_cluster.aks_identity_principal_id } resource "azurerm_role_assignment" "acr_pull_assignment" { - role_definition_name = "AcrPull" - scope = module.container_registry.id - principal_id = module.aks_cluster.kubelet_identity_object_id - skip_service_principal_aad_check = true + role_definition_name = "AcrPull" + scope = module.container_registry.id + principal_id = module.aks_cluster.kubelet_identity_object_id } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index c5e896885..50892d17b 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -25,8 +25,8 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { name = "system" node_count = 1 vm_size = var.system_node_pool_vm_size - vnet_subnet_id = module.virtual_network.subnet_ids[var.system_node_pool_subnet_name] - pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] + vnet_subnet_id = var.system_node_pool_subnet_name + pod_subnet_id = var.pod_subnet_name zones = ["1", "2", "3"] max_pods = 50 os_disk_type = "Ephemeral" @@ -66,8 +66,8 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { vm_size = var.user_node_pool_vm_size mode = "User" zones = ["1", "2", "3"] - vnet_subnet_id = module.virtual_network.subnet_ids[var.user_node_pool_subnet_name] - pod_subnet_id = module.virtual_network.subnet_ids[var.pod_subnet_name] + vnet_subnet_id = var.user_node_pool_subnet_name + pod_subnet_id = var.pod_subnet_name orchestrator_version = var.kubernetes_version max_pods = 50 os_disk_type = "Ephemeral" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf index 1cdba4e09..6f6a0f76a 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf @@ -14,6 +14,10 @@ variable "location" { type = string } +variable "tenant_id" { + type = string +} + variable "kubernetes_version" { type = string } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf new file mode 100644 index 000000000..c4bb3d273 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf @@ -0,0 +1,7 @@ +output name { + value = azurerm_container_registry.acr.name +} + +output id { + value = azurerm_container_registry.acr.id +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf new file mode 100644 index 000000000..ebd280be9 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf @@ -0,0 +1,3 @@ +output name { + value = azurerm_storage_account.storage_account.name +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf index 879aad9c4..af0cdc680 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf @@ -1,5 +1,5 @@ resource "azurerm_virtual_network" "vnet" { - name = var.vnet_name + name = var.name address_space = var.address_space location = var.location resource_group_name = var.resource_group_name @@ -12,8 +12,8 @@ resource "azurerm_subnet" "subnet" { resource_group_name = var.resource_group_name virtual_network_name = azurerm_virtual_network.vnet.name address_prefixes = each.value.address_prefixes - private_endpoint_network_policies = each.value.private_endpoint_network_policies - private_link_service_network_policies_enabled = each.value.private_link_service_network_policies_enabled + private_endpoint_network_policies = "Enabled" + private_link_service_network_policies_enabled = false dynamic "delegation" { for_each = each.value.delegation != null ? [each.value.delegation] : [] diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf new file mode 100644 index 000000000..8a6f752a0 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf @@ -0,0 +1,7 @@ +output "name" { + value = azurerm_virtual_network.vnet.name +} + +output "subnet_ids" { + value = azurerm_subnet.subnet.*.id +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf index 1e37598b1..973ab5f81 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf @@ -1,30 +1,24 @@ -variable "resource_group_name" { - description = "Resource Group name" +variable "name" { type = string } variable "location" { - description = "Location in which to deploy the network" type = string } -variable "vnet_name" { - description = "VNET name" +variable "resource_group_name" { type = string } variable "address_space" { - description = "VNET address space" type = list(string) } variable "subnets" { description = "Subnets configuration" type = list(object({ - name = string - address_prefixes = list(string) - private_endpoint_network_policies = string - private_link_service_network_policies_enabled = bool + name = string + address_prefixes = list(string) delegation = object({ name = string, service_delegation = object({ name = string actions = list(string) @@ -33,6 +27,5 @@ variable "subnets" { } variable "log_analytics_workspace_id" { - description = "Specifies the log analytics workspace id" type = string } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index a5f4e45ef..7d6add071 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -1,14 +1,14 @@ variable "name_prefix" { - type = string + type = string default = "AksOpenAiTerraform" } variable "location" { - default = "westus2" type = string + default = "westus2" } variable "email" { - type = string - default = "paolos@microsoft.com" + type = string + default = "paolos@microsoft.com" } \ No newline at end of file From 13ab276f4c71de0a000db58cf170a6efdb7d17f6 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Thu, 16 Jan 2025 16:42:49 -0800 Subject: [PATCH 036/119] Format --- scenarios/AksOpenAiTerraform/terraform/main.tf | 12 ++++++------ .../terraform/modules/container_registry/outputs.tf | 8 ++++---- .../terraform/modules/storage_account/outputs.tf | 4 ++-- .../terraform/modules/virtual_network/outputs.tf | 2 +- .../terraform/modules/virtual_network/variables.tf | 10 +++++----- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 19fc006ed..22872dcac 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -30,22 +30,22 @@ locals { { name : local.system_node_pool_subnet_name address_prefixes : ["10.240.0.0/16"] - delegation = null + delegation = null }, { name : local.user_node_pool_subnet_name address_prefixes : ["10.241.0.0/16"] - delegation = null + delegation = null }, { name : local.vm_subnet_name address_prefixes : ["10.243.1.0/24"] - delegation = null + delegation = null }, { name : "AzureBastionSubnet" address_prefixes : ["10.243.2.0/24"] - delegation = null + delegation = null }, { name : local.pod_subnet_name @@ -90,7 +90,7 @@ module "openai" { location = var.location resource_group_name = azurerm_resource_group.rg.name - sku_name = "S0" + sku_name = "S0" deployments = [ { name = "gpt-35-turbo" @@ -113,7 +113,7 @@ module "aks_cluster" { location = var.location resource_group_name = azurerm_resource_group.rg.name resource_group_id = azurerm_resource_group.rg.id - tenant_id = data.azurerm_client_config.current.tenant_id + tenant_id = data.azurerm_client_config.current.tenant_id kubernetes_version = "1.32" sku_tier = "Free" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf index c4bb3d273..9642edb0a 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf @@ -1,7 +1,7 @@ -output name { - value = azurerm_container_registry.acr.name +output "name" { + value = azurerm_container_registry.acr.name } -output id { - value = azurerm_container_registry.acr.id +output "id" { + value = azurerm_container_registry.acr.id } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf index ebd280be9..d6bc48f53 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf @@ -1,3 +1,3 @@ -output name { - value = azurerm_storage_account.storage_account.name +output "name" { + value = azurerm_storage_account.storage_account.name } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf index 8a6f752a0..cac0aaa53 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf @@ -1,5 +1,5 @@ output "name" { - value = azurerm_virtual_network.vnet.name + value = azurerm_virtual_network.vnet.name } output "subnet_ids" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf index 973ab5f81..c4a844fcb 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf @@ -1,17 +1,17 @@ variable "name" { - type = string + type = string } variable "location" { - type = string + type = string } variable "resource_group_name" { - type = string + type = string } variable "address_space" { - type = list(string) + type = list(string) } variable "subnets" { @@ -27,5 +27,5 @@ variable "subnets" { } variable "log_analytics_workspace_id" { - type = string + type = string } \ No newline at end of file From 27e463c3e34bce9ace801f3125c6bc810bab6629 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Fri, 17 Jan 2025 09:11:52 -0800 Subject: [PATCH 037/119] Fixes --- .../terraform/.terraform.lock.hcl | 19 ---- .../AksOpenAiTerraform/terraform/main.tf | 96 +++++++++---------- .../terraform/modules/aks/main.tf | 10 +- .../terraform/modules/aks/outputs.tf | 19 ++++ .../terraform/modules/aks/variables.tf | 6 +- .../modules/private_dns_zone/main.tf | 13 --- .../modules/storage_account/outputs.tf | 4 +- .../modules/virtual_network/outputs.tf | 2 +- .../AksOpenAiTerraform/terraform/variables.tf | 15 +++ 9 files changed, 93 insertions(+), 91 deletions(-) create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl index 6b63a37e1..2aeb47adf 100644 --- a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl +++ b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl @@ -1,25 +1,6 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/azure/azapi" { - version = "2.2.0" - hashes = [ - "h1:ng+uFmo5IvLRJEVU/sEN81JO9HB32WOtKQT4rM7L/Ic=", - "zh:062be5d8272cac297a88c2057449f449ea6906c4121ba3dfdeb5cecb3ff91178", - "zh:1fd9abec3ffcbf8d0244408334e9bfc8f49ada50978cd73ee0ed5f8560987267", - "zh:48e84b0302af99d7e7f4248a724088fb1c34aeee78c9ca63ec5a9464ec5054a0", - "zh:4e7302883fd9dd83bfbbcd72ebd55f83d8b16ccc6d12d1573d578058e604d5cf", - "zh:5b6e181e32cbf62f5d2ce34f9d6d9ffe17192e24943450bbe335e1baf0494e66", - "zh:62d525d426c6d5f10109ab04a9abc231b204ea413238f5690f69b420a8b8583a", - "zh:90aab23497ec9c7af44ad9ea1a1d6063dc3331334915e1c549527a73c2c6948d", - "zh:91ecf30a01df5e832191e0c55c87f8403a1f584796fd70f9c9c913d35c2e2a37", - "zh:bc3a5db5e4b9695a69dff47cf1e7184eaf5564d3dc50f231cbcbf535dd140d19", - "zh:cb566bec2676511bf4722e24d0dfc9bf58aff78af38b8e0864970f20d263118f", - "zh:d4fa0c1462b389cee313e1c152e00f5dfc175a1be3615d3b23b526a8581e39a5", - "zh:f8136b0f41045a1e5a6dedc6b6fb055faee3d825f84a3192312e3ac5d057ff72", - ] -} - provider "registry.terraform.io/hashicorp/azurerm" { version = "4.16.0" constraints = "~> 4.16.0" diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 22872dcac..d8d6b6dbd 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -15,57 +15,24 @@ data "azurerm_client_config" "current" { } locals { - log_analytics_workspace_name = "Workspace" - log_analytics_retention_days = 30 - + vm_subnet_name = "VmSubnet" system_node_pool_subnet_name = "SystemSubnet" user_node_pool_subnet_name = "UserSubnet" pod_subnet_name = "PodSubnet" - vm_subnet_name = "VmSubnet" namespace = "magic8ball" service_account_name = "magic8ball-sa" - subnets = [ - { - name : local.system_node_pool_subnet_name - address_prefixes : ["10.240.0.0/16"] - delegation = null - }, - { - name : local.user_node_pool_subnet_name - address_prefixes : ["10.241.0.0/16"] - delegation = null - }, - { - name : local.vm_subnet_name - address_prefixes : ["10.243.1.0/24"] - delegation = null - }, - { - name : "AzureBastionSubnet" - address_prefixes : ["10.243.2.0/24"] - delegation = null - }, - { - name : local.pod_subnet_name - address_prefixes : ["10.242.0.0/16"] - delegation = { - name = "delegation" - service_delegation = { - name = "Microsoft.ContainerService/managedClusters" - actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] - } - } - }, - ] + log_analytics_workspace_name = "Workspace" + log_analytics_retention_days = 30 } resource "random_string" "rg_suffix" { length = 6 special = false + lower = false upper = false - numeric = false + numeric = true } resource "random_string" "storage_account_suffix" { @@ -77,7 +44,7 @@ resource "random_string" "storage_account_suffix" { } resource "azurerm_resource_group" "rg" { - name = "${var.name_prefix}-${random_string.rg_suffix}-rg" + name = "${var.name_prefix}-${random_string.rg_suffix.result}-rg" location = var.location } @@ -115,11 +82,12 @@ module "aks_cluster" { resource_group_id = azurerm_resource_group.rg.id tenant_id = data.azurerm_client_config.current.tenant_id - kubernetes_version = "1.32" - sku_tier = "Free" - user_node_pool_subnet_name = local.user_node_pool_subnet_name - system_node_pool_subnet_name = local.system_node_pool_subnet_name - pod_subnet_name = local.pod_subnet_name + kubernetes_version = "1.30.7" + sku_tier = "Free" + + system_node_pool_subnet_id = module.virtual_network.subnet_ids[local.system_node_pool_subnet_name] + user_node_pool_subnet_id = module.virtual_network.subnet_ids[local.user_node_pool_subnet_name] + pod_subnet_id = module.virtual_network.subnet_ids[local.pod_subnet_name] log_analytics_workspace_id = module.log_analytics_workspace.id @@ -154,7 +122,7 @@ module "storage_account" { module "key_vault" { source = "./modules/key_vault" - name = "${var.name_prefix}KeyVault" + name = "${var.name_prefix}Vault" location = var.location resource_group_name = azurerm_resource_group.rg.name @@ -221,7 +189,39 @@ module "virtual_network" { log_analytics_workspace_id = module.log_analytics_workspace.id address_space = ["10.0.0.0/8"] - subnets = local.subnets + subnets = [ + { + name : local.system_node_pool_subnet_name + address_prefixes : ["10.240.0.0/16"] + delegation = null + }, + { + name : local.user_node_pool_subnet_name + address_prefixes : ["10.241.0.0/16"] + delegation = null + }, + { + name : local.vm_subnet_name + address_prefixes : ["10.243.1.0/24"] + delegation = null + }, + { + name : "AzureBastionSubnet" + address_prefixes : ["10.243.2.0/24"] + delegation = null + }, + { + name : local.pod_subnet_name + address_prefixes : ["10.242.0.0/16"] + delegation = { + name = "delegation" + service_delegation = { + name = "Microsoft.ContainerService/managedClusters" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } + }, + ] } module "nat_gateway" { @@ -230,7 +230,7 @@ module "nat_gateway" { location = var.location resource_group_name = azurerm_resource_group.rg.name - subnet_ids = module.virtual_network.subnet_ids[local.system_node_pool_subnet_name] + subnet_ids = module.virtual_network.subnet_ids } module "bastion_host" { @@ -344,7 +344,7 @@ module "blob_private_endpoint" { location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] - private_connection_resource_id = module.storage_account.name + private_connection_resource_id = module.storage_account.id is_manual_connection = false subresource_name = "blob" private_dns_zone_group_name = "BlobPrivateDnsZoneGroup" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index 50892d17b..d775d5a54 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -25,8 +25,8 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { name = "system" node_count = 1 vm_size = var.system_node_pool_vm_size - vnet_subnet_id = var.system_node_pool_subnet_name - pod_subnet_id = var.pod_subnet_name + vnet_subnet_id = var.system_node_pool_subnet_id + pod_subnet_id = var.pod_subnet_id zones = ["1", "2", "3"] max_pods = 50 os_disk_type = "Ephemeral" @@ -50,7 +50,7 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { } azure_active_directory_role_based_access_control { - tenant_id = data.azurerm_client_config.current.tenant_id + tenant_id = var.tenant_id azure_rbac_enabled = true } @@ -66,8 +66,8 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { vm_size = var.user_node_pool_vm_size mode = "User" zones = ["1", "2", "3"] - vnet_subnet_id = var.user_node_pool_subnet_name - pod_subnet_id = var.pod_subnet_name + vnet_subnet_id = var.user_node_pool_subnet_id + pod_subnet_id = var.pod_subnet_id orchestrator_version = var.kubernetes_version max_pods = 50 os_disk_type = "Ephemeral" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf new file mode 100644 index 000000000..56139a135 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf @@ -0,0 +1,19 @@ +output "name" { + value = azurerm_kubernetes_cluster.aks_cluster.name +} + +output "id" { + value = azurerm_kubernetes_cluster.aks_cluster.id +} + +output "aks_identity_principal_id" { + value = azurerm_user_assigned_identity.aks_identity.principal_id +} + +output "kubelet_identity_object_id" { + value = azurerm_kubernetes_cluster.aks_cluster.kubelet_identity.0.object_id +} + +output "oidc_issuer_url" { + value = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf index 6f6a0f76a..74e3a7ca5 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf @@ -40,14 +40,14 @@ variable "log_analytics_workspace_id" { type = string } -variable "user_node_pool_subnet_name" { +variable "user_node_pool_subnet_id" { type = string } -variable "system_node_pool_subnet_name" { +variable "system_node_pool_subnet_id" { type = string } -variable "pod_subnet_name" { +variable "pod_subnet_id" { type = string } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf index fb97cc407..be1d6a7ea 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf @@ -1,13 +1,6 @@ resource "azurerm_private_dns_zone" "private_dns_zone" { name = var.name resource_group_name = var.resource_group_name - tags = var.tags - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_private_dns_zone_virtual_network_link" "link" { @@ -17,10 +10,4 @@ resource "azurerm_private_dns_zone_virtual_network_link" "link" { resource_group_name = var.resource_group_name private_dns_zone_name = azurerm_private_dns_zone.private_dns_zone.name virtual_network_id = "/subscriptions/${each.value.subscription_id}/resourceGroups/${each.value.resource_group_name}/providers/Microsoft.Network/virtualNetworks/${each.key}" - - lifecycle { - ignore_changes = [ - tags - ] - } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf index d6bc48f53..156c1d8d7 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf @@ -1,3 +1,3 @@ -output "name" { - value = azurerm_storage_account.storage_account.name +output "id" { + value = azurerm_storage_account.storage_account.id } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf index cac0aaa53..b8d3adc64 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf @@ -3,5 +3,5 @@ output "name" { } output "subnet_ids" { - value = azurerm_subnet.subnet.*.id + value = { for subnet in azurerm_subnet.subnet : subnet.name => subnet.id } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 7d6add071..469b78345 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -8,6 +8,21 @@ variable "location" { default = "westus2" } +variable "kubernetes_version" { + type = string + default = "1.30.7" +} + +variable "system_node_pool_vm_size" { + type = string + default = "Standard_D8ds_v5" +} + +variable "user_node_pool_vm_size" { + type = string + default = "Standard_D8ds_v5" +} + variable "email" { type = string default = "paolos@microsoft.com" From 9710cf524c8ca3a338d98f7a9b6d7d8f8b3efb53 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Fri, 17 Jan 2025 09:33:05 -0800 Subject: [PATCH 038/119] Fixes --- scenarios/AksOpenAiTerraform/README.md | 24 ++-------- .../AksOpenAiTerraform/terraform/main.tf | 16 +++---- .../terraform/modules/aks/variables.tf | 6 +-- .../modules/private_endpoint/main.tf | 5 +- .../modules/private_endpoint/variables.tf | 46 ++++--------------- 5 files changed, 23 insertions(+), 74 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index 0d8378ae4..e670135fc 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -8,8 +8,6 @@ ms.author: ariaamini ms.custom: innovation-engine, linux-related-content --- - - ## Install AKS extension Run commands below to set up AKS extensions for Azure. @@ -18,30 +16,16 @@ Run commands below to set up AKS extensions for Azure. ./terraform/register-preview-features.sh ``` -## Set up service principal - -A Service Principal is an application within Azure Active Directory with the authentication tokens Terraform needs to perform actions on your behalf. - -```bash -# TODO: fix -# az ad sp create-for-rbac --role="Contributor" --scopes="/subscriptions/$ARM_SUBSCRIPTION_ID" -``` +## Set up Subscription ID to authenticate for Terraform -## Setup Infra +Terraform uses the ARM_SUBSCRIPTION_ID environment variable to authenticate while using CLI. ```bash export ARM_SUBSCRIPTION_ID="0c8875c7-e423-4caa-827a-1f0350bd8dd3" -# For debugging in powershell -# $env:ARM_SUBSCRIPTION_ID = "0c8875c7-e423-4caa-827a-1f0350bd8dd3" - -terraform apply ``` -## Set up environment +## Run Terraform ```bash -export ARM_CLIENT_ID="" -export ARM_CLIENT_SECRET="" -export ARM_SUBSCRIPTION_ID="" -export ARM_TENANT_ID="" +terraform apply ``` diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index d8d6b6dbd..2339ea1ab 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -82,8 +82,10 @@ module "aks_cluster" { resource_group_id = azurerm_resource_group.rg.id tenant_id = data.azurerm_client_config.current.tenant_id - kubernetes_version = "1.30.7" - sku_tier = "Free" + kubernetes_version = var.kubernetes_version + sku_tier = "Free" + system_node_pool_vm_size = var.system_node_pool_vm_size + user_node_pool_vm_size = var.user_node_pool_vm_size system_node_pool_subnet_id = module.virtual_network.subnet_ids[local.system_node_pool_subnet_name] user_node_pool_subnet_id = module.virtual_network.subnet_ids[local.user_node_pool_subnet_name] @@ -103,10 +105,10 @@ module "container_registry" { location = var.location resource_group_name = azurerm_resource_group.rg.name - log_analytics_workspace_id = module.log_analytics_workspace.id - - sku = "Basic" + sku = "Premium" admin_enabled = true + + log_analytics_workspace_id = module.log_analytics_workspace.id } module "storage_account" { @@ -306,7 +308,6 @@ module "openai_private_endpoint" { resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.openai.id - is_manual_connection = false subresource_name = "account" private_dns_zone_group_name = "AcrPrivateDnsZoneGroup" private_dns_zone_group_ids = [module.openai_private_dns_zone.id] @@ -319,7 +320,6 @@ module "acr_private_endpoint" { resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.container_registry.id - is_manual_connection = false subresource_name = "registry" private_dns_zone_group_name = "AcrPrivateDnsZoneGroup" private_dns_zone_group_ids = [module.acr_private_dns_zone.id] @@ -332,7 +332,6 @@ module "key_vault_private_endpoint" { resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.key_vault.id - is_manual_connection = false subresource_name = "vault" private_dns_zone_group_name = "KeyVaultPrivateDnsZoneGroup" private_dns_zone_group_ids = [module.key_vault_private_dns_zone.id] @@ -345,7 +344,6 @@ module "blob_private_endpoint" { resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.storage_account.id - is_manual_connection = false subresource_name = "blob" private_dns_zone_group_name = "BlobPrivateDnsZoneGroup" private_dns_zone_group_ids = [module.blob_private_dns_zone.id] diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf index 74e3a7ca5..c0e76833b 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf @@ -27,13 +27,11 @@ variable "sku_tier" { } variable "system_node_pool_vm_size" { - default = "Standard_D8ds_v5" - type = string + type = string } variable "user_node_pool_vm_size" { - default = "Standard_D8ds_v5" - type = string + type = string } variable "log_analytics_workspace_id" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf index 2b9b78868..c73bdaefd 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf @@ -7,9 +7,8 @@ resource "azurerm_private_endpoint" "private_endpoint" { private_service_connection { name = "${var.name}Connection" private_connection_resource_id = var.private_connection_resource_id - is_manual_connection = var.is_manual_connection - subresource_names = try([var.subresource_name], null) - request_message = try(var.request_message, null) + is_manual_connection = false + subresource_names = [var.subresource_name] } private_dns_zone_group { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf index f7a410572..8bc78cbef 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf @@ -1,61 +1,31 @@ variable "name" { - description = "(Required) Specifies the name of the private endpoint. Changing this forces a new resource to be created." - type = string + type = string } variable "resource_group_name" { - description = "(Required) The name of the resource group. Changing this forces a new resource to be created." - type = string + type = string } variable "private_connection_resource_id" { - description = "(Required) Specifies the resource id of the private link service" - type = string + type = string } variable "location" { - description = "(Required) Specifies the supported Azure location where the resource exists. Changing this forces a new resource to be created." - type = string + type = string } variable "subnet_id" { - description = "(Required) Specifies the resource id of the subnet" - type = string -} - -variable "is_manual_connection" { - description = "(Optional) Specifies whether the private endpoint connection requires manual approval from the remote resource owner." - type = string - default = false + type = string } variable "subresource_name" { - description = "(Optional) Specifies a subresource name which the Private Endpoint is able to connect to." - type = string - default = null -} - -variable "request_message" { - description = "(Optional) Specifies a message passed to the owner of the remote resource when the private endpoint attempts to establish the connection to the remote resource." - type = string - default = null + type = string } variable "private_dns_zone_group_name" { - description = "(Required) Specifies the Name of the Private DNS Zone Group. Changing this forces a new private_dns_zone_group resource to be created." - type = string + type = string } variable "private_dns_zone_group_ids" { - description = "(Required) Specifies the list of Private DNS Zones to include within the private_dns_zone_group." - type = list(string) -} - -variable "tags" { - description = "(Optional) Specifies the tags of the network security group" - default = {} -} - -variable "private_dns" { - default = {} + type = list(string) } \ No newline at end of file From 2aac95f2335053ff994ac0a13896c908655a5a43 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Fri, 17 Jan 2025 09:54:25 -0800 Subject: [PATCH 039/119] Extract var --- scenarios/AksOpenAiTerraform/terraform/main.tf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 2339ea1ab..7a51561a3 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -15,6 +15,8 @@ data "azurerm_client_config" "current" { } locals { + tenant_id = data.azurerm_client_config.current.tenant_id + vm_subnet_name = "VmSubnet" system_node_pool_subnet_name = "SystemSubnet" user_node_pool_subnet_name = "UserSubnet" @@ -80,7 +82,7 @@ module "aks_cluster" { location = var.location resource_group_name = azurerm_resource_group.rg.name resource_group_id = azurerm_resource_group.rg.id - tenant_id = data.azurerm_client_config.current.tenant_id + tenant_id = local.tenant_id kubernetes_version = var.kubernetes_version sku_tier = "Free" @@ -128,7 +130,7 @@ module "key_vault" { location = var.location resource_group_name = azurerm_resource_group.rg.name - tenant_id = data.azurerm_client_config.current.tenant_id + tenant_id = local.tenant_id sku_name = "standard" enabled_for_deployment = true enabled_for_disk_encryption = true @@ -148,7 +150,7 @@ module "deployment_script" { location = var.location resource_group_name = azurerm_resource_group.rg.name - azure_cli_version = "2.9.1" + azure_cli_version = "2.68.0" managed_identity_name = "${var.name_prefix}ScriptManagedIdentity" aks_cluster_name = module.aks_cluster.name hostname = "magic8ball.contoso.com" @@ -156,7 +158,7 @@ module "deployment_script" { service_account_name = local.service_account_name email = var.email primary_script_uri = "https://paolosalvatori.blob.core.windows.net/scripts/install-nginx-via-helm-and-create-sa.sh" - tenant_id = data.azurerm_client_config.current.tenant_id + tenant_id = local.tenant_id subscription_id = data.azurerm_client_config.current.subscription_id workload_managed_identity_client_id = azurerm_user_assigned_identity.aks_workload_identity.client_id From 0c0f858bc150e1b5fe2570e5b88f6420f9275d90 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Fri, 17 Jan 2025 09:57:23 -0800 Subject: [PATCH 040/119] Fix vars --- .../terraform/modules/openai/main.tf | 7 --- .../terraform/modules/openai/variables.tf | 45 ++++--------------- 2 files changed, 9 insertions(+), 43 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf index 235dca40d..0d8965ba0 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf @@ -6,17 +6,10 @@ resource "azurerm_cognitive_account" "openai" { custom_subdomain_name = var.custom_subdomain_name sku_name = var.sku_name public_network_access_enabled = var.public_network_access_enabled - tags = var.tags identity { type = "SystemAssigned" } - - lifecycle { - ignore_changes = [ - tags - ] - } } resource "azurerm_cognitive_deployment" "deployment" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf index dca286ff8..1a27b84b5 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf @@ -1,43 +1,29 @@ variable "resource_group_name" { - description = "(Required) Specifies the resource group name" - type = string + type = string } variable "location" { - description = "(Required) Specifies the location of the Azure OpenAI Service" - type = string + type = string } variable "name" { - description = "(Required) Specifies the name of the Azure OpenAI Service" - type = string + type = string } variable "sku_name" { - description = "(Optional) Specifies the sku name for the Azure OpenAI Service" - type = string - default = "S0" -} - -variable "tags" { - description = "(Optional) Specifies the tags of the Azure OpenAI Service" - type = map(any) - default = {} + type = string } variable "custom_subdomain_name" { - description = "(Optional) Specifies the custom subdomain name of the Azure OpenAI Service" - type = string + type = string } variable "public_network_access_enabled" { - description = "(Optional) Specifies whether public network access is allowed for the Azure OpenAI Service" - type = bool - default = true + type = bool + default = true } variable "deployments" { - description = "(Optional) Specifies the deployments of the Azure OpenAI Service" type = list(object({ name = string model = object({ @@ -46,25 +32,12 @@ variable "deployments" { }) rai_policy_name = string })) - default = [ - { - name = "gpt-35-turbo" - model = { - name = "gpt-35-turbo" - version = "0301" - } - rai_policy_name = "" - } - ] } variable "log_analytics_workspace_id" { - description = "Specifies the log analytics workspace id" - type = string + type = string } variable "log_analytics_retention_days" { - description = "Specifies the number of days of the retention policy" - type = number - default = 7 + type = number } \ No newline at end of file From dd3b1317c39e33a9a2762445bf400ffec8d3e04d Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Fri, 17 Jan 2025 10:12:39 -0800 Subject: [PATCH 041/119] Update openai model + cleanup --- .../AksOpenAiTerraform/terraform/main.tf | 9 ++-- .../terraform/modules/openai/variables.tf | 1 - .../terraform/modules/storage_account/main.tf | 7 --- .../modules/storage_account/variables.tf | 54 ------------------- 4 files changed, 5 insertions(+), 66 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 7a51561a3..10bf88430 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -62,16 +62,16 @@ module "openai" { sku_name = "S0" deployments = [ { - name = "gpt-35-turbo" + name = "gpt-4" model = { - name = "gpt-35-turbo" - version = "0301" + name = "gpt-4" + version = "turbo-2024-04-09" } - rai_policy_name = "" } ] custom_subdomain_name = lower("${var.name_prefix}OpenAi") public_network_access_enabled = true + log_analytics_workspace_id = module.log_analytics_workspace.id log_analytics_retention_days = local.log_analytics_retention_days } @@ -122,6 +122,7 @@ module "storage_account" { account_kind = "StorageV2" account_tier = "Standard" replication_type = "LRS" + is_hns_enabled = false } module "key_vault" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf index 1a27b84b5..9bb21252d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf @@ -30,7 +30,6 @@ variable "deployments" { name = string version = string }) - rai_policy_name = string })) } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf index a54ed2f26..6e885b845 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf @@ -7,16 +7,9 @@ resource "azurerm_storage_account" "storage_account" { account_tier = var.account_tier account_replication_type = var.replication_type is_hns_enabled = var.is_hns_enabled - tags = var.tags allow_nested_items_to_be_public = false - network_rules { - default_action = (length(var.ip_rules) + length(var.virtual_network_subnet_ids)) > 0 ? "Deny" : var.default_action - ip_rules = var.ip_rules - virtual_network_subnet_ids = var.virtual_network_subnet_ids - } - identity { type = "SystemAssigned" } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf index b38fcad5a..9c1a110e3 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf @@ -1,81 +1,27 @@ variable "resource_group_name" { - description = "(Required) Specifies the resource group name of the storage account" type = string } variable "name" { - description = "(Required) Specifies the name of the storage account" type = string } variable "location" { - description = "(Required) Specifies the location of the storage account" type = string } variable "account_kind" { - description = "(Optional) Specifies the account kind of the storage account" - default = "StorageV2" type = string - - validation { - condition = contains(["Storage", "StorageV2"], var.account_kind) - error_message = "The account kind of the storage account is invalid." - } } variable "account_tier" { - description = "(Optional) Specifies the account tier of the storage account" - default = "Standard" type = string - - validation { - condition = contains(["Standard", "Premium"], var.account_tier) - error_message = "The account tier of the storage account is invalid." - } } variable "replication_type" { - description = "(Optional) Specifies the replication type of the storage account" - default = "LRS" type = string - - validation { - condition = contains(["LRS", "ZRS", "GRS", "GZRS", "RA-GRS", "RA-GZRS"], var.replication_type) - error_message = "The replication type of the storage account is invalid." - } } variable "is_hns_enabled" { - description = "(Optional) Specifies the replication type of the storage account" - default = false type = bool -} - -variable "default_action" { - description = "Allow or disallow public access to all blobs or containers in the storage accounts. The default interpretation is true for this property." - default = "Allow" - type = string -} - -variable "ip_rules" { - description = "Specifies IP rules for the storage account" - default = [] - type = list(string) -} - -variable "virtual_network_subnet_ids" { - description = "Specifies a list of resource ids for subnets" - default = [] - type = list(string) -} - -variable "kind" { - description = "(Optional) Specifies the kind of the storage account" - default = "" -} - -variable "tags" { - description = "(Optional) Specifies the tags of the storage account" - default = {} } \ No newline at end of file From b9ac095d4356db6de5976137ce117feb97766cc9 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Fri, 17 Jan 2025 11:17:42 -0800 Subject: [PATCH 042/119] Fixes --- scenarios/AksOpenAiTerraform/README.md | 6 ++ .../AksOpenAiTerraform/terraform/main.tf | 73 ++++++++++--------- .../terraform/modules/aks/main.tf | 6 ++ .../modules/deployment_script/main.tf | 10 +-- .../modules/deployment_script/variables.tf | 26 +------ .../terraform/modules/openai/main.tf | 1 + .../modules/storage_account/variables.tf | 14 ++-- .../AksOpenAiTerraform/terraform/variables.tf | 2 +- 8 files changed, 67 insertions(+), 71 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index e670135fc..f4aec438f 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -24,6 +24,12 @@ Terraform uses the ARM_SUBSCRIPTION_ID environment variable to authenticate whil export ARM_SUBSCRIPTION_ID="0c8875c7-e423-4caa-827a-1f0350bd8dd3" ``` +## Init Terraform + +```bash +terraform init +``` + ## Run Terraform ```bash diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 10bf88430..d5412da35 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -14,8 +14,18 @@ provider "azurerm" { data "azurerm_client_config" "current" { } +resource "random_string" "rg_suffix" { + length = 6 + special = false + lower = false + upper = false + numeric = true +} + locals { tenant_id = data.azurerm_client_config.current.tenant_id + subscription_id = data.azurerm_client_config.current.subscription_id + random_id = random_string.rg_suffix.result vm_subnet_name = "VmSubnet" system_node_pool_subnet_name = "SystemSubnet" @@ -29,14 +39,6 @@ locals { log_analytics_retention_days = 30 } -resource "random_string" "rg_suffix" { - length = 6 - special = false - lower = false - upper = false - numeric = true -} - resource "random_string" "storage_account_suffix" { length = 8 special = false @@ -46,8 +48,12 @@ resource "random_string" "storage_account_suffix" { } resource "azurerm_resource_group" "rg" { - name = "${var.name_prefix}-${random_string.rg_suffix.result}-rg" + name = "${var.resource_group_name_prefix}-${local.random_id}-rg" location = var.location + + lifecycle { + ignore_changes = [tags] + } } ############################################################################### @@ -55,7 +61,7 @@ resource "azurerm_resource_group" "rg" { ############################################################################### module "openai" { source = "./modules/openai" - name = "${var.name_prefix}OpenAi" + name = "OpenAi-${local.random_id}" location = var.location resource_group_name = azurerm_resource_group.rg.name @@ -69,16 +75,16 @@ module "openai" { } } ] - custom_subdomain_name = lower("${var.name_prefix}OpenAi") + custom_subdomain_name = "magic8ball" public_network_access_enabled = true - - log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = local.log_analytics_retention_days + + log_analytics_workspace_id = module.log_analytics_workspace.id + log_analytics_retention_days = local.log_analytics_retention_days } module "aks_cluster" { source = "./modules/aks" - name = "${var.name_prefix}AksCluster" + name = "AksCluster" location = var.location resource_group_name = azurerm_resource_group.rg.name resource_group_id = azurerm_resource_group.rg.id @@ -103,7 +109,7 @@ module "aks_cluster" { module "container_registry" { source = "./modules/container_registry" - name = "${var.name_prefix}Acr" + name = "azure-container-registry" location = var.location resource_group_name = azurerm_resource_group.rg.name @@ -127,7 +133,7 @@ module "storage_account" { module "key_vault" { source = "./modules/key_vault" - name = "${var.name_prefix}Vault" + name = "KeyVault-${local.random_id}" location = var.location resource_group_name = azurerm_resource_group.rg.name @@ -147,12 +153,13 @@ module "key_vault" { module "deployment_script" { source = "./modules/deployment_script" - name = "${var.name_prefix}BashScript" + name = "DeployBashScript" location = var.location resource_group_name = azurerm_resource_group.rg.name - azure_cli_version = "2.68.0" - managed_identity_name = "${var.name_prefix}ScriptManagedIdentity" + azure_cli_version = "2.64.0" + aks_cluster_id = module.aks_cluster.id + managed_identity_name = "ScriptManagedIdentity" aks_cluster_name = module.aks_cluster.name hostname = "magic8ball.contoso.com" namespace = local.namespace @@ -160,7 +167,7 @@ module "deployment_script" { email = var.email primary_script_uri = "https://paolosalvatori.blob.core.windows.net/scripts/install-nginx-via-helm-and-create-sa.sh" tenant_id = local.tenant_id - subscription_id = data.azurerm_client_config.current.subscription_id + subscription_id = local.subscription_id workload_managed_identity_client_id = azurerm_user_assigned_identity.aks_workload_identity.client_id depends_on = [ @@ -170,7 +177,7 @@ module "deployment_script" { module "log_analytics_workspace" { source = "./modules/log_analytics" - name = "${var.name_prefix}${local.log_analytics_workspace_name}" + name = "${local.log_analytics_workspace_name}" location = var.location resource_group_name = azurerm_resource_group.rg.name @@ -231,7 +238,7 @@ module "virtual_network" { module "nat_gateway" { source = "./modules/nat_gateway" - name = "${var.name_prefix}NatGateway" + name = "NatGateway" location = var.location resource_group_name = azurerm_resource_group.rg.name @@ -240,7 +247,7 @@ module "nat_gateway" { module "bastion_host" { source = "./modules/bastion_host" - name = "${var.name_prefix}BastionHost" + name = "BastionHost" location = var.location resource_group_name = azurerm_resource_group.rg.name @@ -259,7 +266,7 @@ module "acr_private_dns_zone" { resource_group_name = azurerm_resource_group.rg.name virtual_networks_to_link = { (module.virtual_network.name) = { - subscription_id = data.azurerm_client_config.current.subscription_id + subscription_id = local.subscription_id resource_group_name = azurerm_resource_group.rg.name } } @@ -271,7 +278,7 @@ module "openai_private_dns_zone" { resource_group_name = azurerm_resource_group.rg.name virtual_networks_to_link = { (module.virtual_network.name) = { - subscription_id = data.azurerm_client_config.current.subscription_id + subscription_id = local.subscription_id resource_group_name = azurerm_resource_group.rg.name } } @@ -283,7 +290,7 @@ module "key_vault_private_dns_zone" { resource_group_name = azurerm_resource_group.rg.name virtual_networks_to_link = { (module.virtual_network.name) = { - subscription_id = data.azurerm_client_config.current.subscription_id + subscription_id = local.subscription_id resource_group_name = azurerm_resource_group.rg.name } } @@ -295,7 +302,7 @@ module "blob_private_dns_zone" { resource_group_name = azurerm_resource_group.rg.name virtual_networks_to_link = { (module.virtual_network.name) = { - subscription_id = data.azurerm_client_config.current.subscription_id + subscription_id = local.subscription_id resource_group_name = azurerm_resource_group.rg.name } } @@ -306,7 +313,7 @@ module "blob_private_dns_zone" { ############################################################################### module "openai_private_endpoint" { source = "./modules/private_endpoint" - name = "${module.openai.name}PrivateEndpoint" + name = "OpenAiPrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] @@ -318,7 +325,7 @@ module "openai_private_endpoint" { module "acr_private_endpoint" { source = "./modules/private_endpoint" - name = "${module.container_registry.name}PrivateEndpoint" + name = "AcrPrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] @@ -330,7 +337,7 @@ module "acr_private_endpoint" { module "key_vault_private_endpoint" { source = "./modules/private_endpoint" - name = "${module.key_vault.name}PrivateEndpoint" + name = "VaultPrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] @@ -342,7 +349,7 @@ module "key_vault_private_endpoint" { module "blob_private_endpoint" { source = "./modules/private_endpoint" - name = "${var.name_prefix}BlobStoragePrivateEndpoint" + name = "BlobStoragePrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] @@ -356,7 +363,7 @@ module "blob_private_endpoint" { # Identities/Roles ############################################################################### resource "azurerm_user_assigned_identity" "aks_workload_identity" { - name = "${var.name_prefix}WorkloadManagedIdentity" + name = "WorkloadManagedIdentity" resource_group_name = azurerm_resource_group.rg.name location = var.location } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index d775d5a54..de4c20227 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -30,6 +30,12 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { zones = ["1", "2", "3"] max_pods = 50 os_disk_type = "Ephemeral" + + upgrade_settings { + drain_timeout_in_minutes = 0 + max_surge = "10%" + node_soak_duration_in_minutes = 0 + } } identity { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf index 38e5cc841..3e6e291eb 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf @@ -4,22 +4,17 @@ resource "azurerm_user_assigned_identity" "script_identity" { resource_group_name = var.resource_group_name } -data "azurerm_kubernetes_cluster" "aks_cluster" { - name = var.aks_cluster_name - resource_group_name = var.resource_group_name -} - resource "azurerm_role_assignment" "network_contributor_assignment" { - scope = data.azurerm_kubernetes_cluster.aks_cluster.id + scope = var.aks_cluster_id role_definition_name = "Azure Kubernetes Service Cluster Admin Role" principal_id = azurerm_user_assigned_identity.script_identity.principal_id - skip_service_principal_aad_check = true } resource "azurerm_resource_deployment_script_azure_cli" "script" { name = var.name resource_group_name = var.resource_group_name location = var.location + version = var.azure_cli_version retention_interval = "P1D" command_line = "'foo' 'bar'" @@ -27,7 +22,6 @@ resource "azurerm_resource_deployment_script_azure_cli" "script" { force_update_tag = "1" timeout = "PT30M" primary_script_uri = var.primary_script_uri - tags = var.tags identity { type = "UserAssigned" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf index f650b86fc..b3f4bd2d1 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf @@ -1,78 +1,60 @@ variable "resource_group_name" { - description = "(Required) Specifies the resource group name" type = string } variable "location" { - description = "(Required) Specifies the location of the Azure OpenAI Service" type = string } variable "name" { - description = "(Required) Specifies the name of the Azure OpenAI Service" type = string - default = "BashScript" +} + +variable "aks_cluster_id" { + type = string } variable "azure_cli_version" { - description = "(Required) Azure CLI module version to be used." type = string - default = "2.9.1" } variable "managed_identity_name" { - description = "Specifies the name of the user-defined managed identity used by the deployment script." type = string - default = "ScriptManagedIdentity" } variable "primary_script_uri" { - description = "(Optional) Uri for the script. This is the entry point for the external script. Changing this forces a new Resource Deployment Script to be created." type = string } variable "aks_cluster_name" { - description = "Specifies the name of the AKS cluster." type = string } variable "tenant_id" { - description = "Specifies the Azure AD tenant id." type = string } variable "subscription_id" { - description = "Specifies the Azure subscription id." type = string } variable "hostname" { - description = "Specifies the hostname of the application." type = string } variable "namespace" { - description = "Specifies the namespace of the application." type = string } variable "service_account_name" { - description = "Specifies the service account of the application." type = string } variable "workload_managed_identity_client_id" { - description = "Specifies the client id of the workload user-defined managed identity." type = string } variable "email" { description = "Specifies the email address for the cert-manager cluster issuer." type = string -} - -variable "tags" { - description = "(Optional) Specifies the tags of the Azure OpenAI Service" - type = map(any) - default = {} } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf index 0d8965ba0..20b8af513 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf @@ -2,6 +2,7 @@ resource "azurerm_cognitive_account" "openai" { name = var.name location = var.location resource_group_name = var.resource_group_name + kind = "OpenAI" custom_subdomain_name = var.custom_subdomain_name sku_name = var.sku_name diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf index 9c1a110e3..dbd9d37c6 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf @@ -1,27 +1,27 @@ variable "resource_group_name" { - type = string + type = string } variable "name" { - type = string + type = string } variable "location" { - type = string + type = string } variable "account_kind" { - type = string + type = string } variable "account_tier" { - type = string + type = string } variable "replication_type" { - type = string + type = string } variable "is_hns_enabled" { - type = bool + type = bool } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 469b78345..2c90c24b0 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -1,4 +1,4 @@ -variable "name_prefix" { +variable "resource_group_name_prefix" { type = string default = "AksOpenAiTerraform" } From 09fa7ada700d73e5d780abd0b91ad3a3f667f0ea Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Fri, 17 Jan 2025 11:36:44 -0800 Subject: [PATCH 043/119] Fix names + region --- scenarios/AksOpenAiTerraform/terraform/main.tf | 2 +- scenarios/AksOpenAiTerraform/terraform/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index d5412da35..fdeaf0ffb 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -109,7 +109,7 @@ module "aks_cluster" { module "container_registry" { source = "./modules/container_registry" - name = "azure-container-registry" + name = "acr${local.random_id}" location = var.location resource_group_name = azurerm_resource_group.rg.name diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 2c90c24b0..5bb8b53d4 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -5,7 +5,7 @@ variable "resource_group_name_prefix" { variable "location" { type = string - default = "westus2" + default = "westus" } variable "kubernetes_version" { From eec104a20507a92b425f497e271f5f45686ccdbc Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Fri, 17 Jan 2025 12:17:25 -0800 Subject: [PATCH 044/119] Clean up --- .../AksOpenAiTerraform/terraform/main.tf | 10 ++- .../modules/deployment_script/main.tf | 20 ++--- .../modules/deployment_script/variables.tf | 28 +++--- .../terraform/modules/key_vault/main.tf | 15 ++-- .../terraform/modules/key_vault/outputs.tf | 6 +- .../terraform/modules/key_vault/variables.tf | 86 ++++--------------- .../terraform/modules/log_analytics/main.tf | 8 +- .../modules/log_analytics/variables.tf | 31 ++----- .../terraform/modules/openai/main.tf | 8 +- .../modules/private_dns_zone/variables.tf | 18 ++-- .../AksOpenAiTerraform/terraform/variables.tf | 2 +- 11 files changed, 72 insertions(+), 160 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index fdeaf0ffb..a2b4632e5 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -23,9 +23,9 @@ resource "random_string" "rg_suffix" { } locals { - tenant_id = data.azurerm_client_config.current.tenant_id + tenant_id = data.azurerm_client_config.current.tenant_id subscription_id = data.azurerm_client_config.current.subscription_id - random_id = random_string.rg_suffix.result + random_id = random_string.rg_suffix.result vm_subnet_name = "VmSubnet" system_node_pool_subnet_name = "SystemSubnet" @@ -158,7 +158,7 @@ module "deployment_script" { resource_group_name = azurerm_resource_group.rg.name azure_cli_version = "2.64.0" - aks_cluster_id = module.aks_cluster.id + aks_cluster_id = module.aks_cluster.id managed_identity_name = "ScriptManagedIdentity" aks_cluster_name = module.aks_cluster.name hostname = "magic8ball.contoso.com" @@ -177,10 +177,12 @@ module "deployment_script" { module "log_analytics_workspace" { source = "./modules/log_analytics" - name = "${local.log_analytics_workspace_name}" + name = local.log_analytics_workspace_name location = var.location resource_group_name = azurerm_resource_group.rg.name + sku = "PerGB2018" + retention_in_days = local.log_analytics_retention_days solution_plan_map = { ContainerInsights = { product = "OMSGallery/ContainerInsights" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf index 3e6e291eb..82d3368ee 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf @@ -5,9 +5,9 @@ resource "azurerm_user_assigned_identity" "script_identity" { } resource "azurerm_role_assignment" "network_contributor_assignment" { - scope = var.aks_cluster_id - role_definition_name = "Azure Kubernetes Service Cluster Admin Role" - principal_id = azurerm_user_assigned_identity.script_identity.principal_id + scope = var.aks_cluster_id + role_definition_name = "Azure Kubernetes Service Cluster Admin Role" + principal_id = azurerm_user_assigned_identity.script_identity.principal_id } resource "azurerm_resource_deployment_script_azure_cli" "script" { @@ -15,13 +15,13 @@ resource "azurerm_resource_deployment_script_azure_cli" "script" { resource_group_name = var.resource_group_name location = var.location - version = var.azure_cli_version - retention_interval = "P1D" - command_line = "'foo' 'bar'" - cleanup_preference = "OnSuccess" - force_update_tag = "1" - timeout = "PT30M" - primary_script_uri = var.primary_script_uri + version = var.azure_cli_version + retention_interval = "P1D" + command_line = "'foo' 'bar'" + cleanup_preference = "OnSuccess" + force_update_tag = "1" + timeout = "PT30M" + primary_script_uri = var.primary_script_uri identity { type = "UserAssigned" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf index b3f4bd2d1..20ac1307e 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf @@ -1,57 +1,57 @@ variable "resource_group_name" { - type = string + type = string } variable "location" { - type = string + type = string } variable "name" { - type = string + type = string } variable "aks_cluster_id" { - type = string + type = string } variable "azure_cli_version" { - type = string + type = string } variable "managed_identity_name" { - type = string + type = string } variable "primary_script_uri" { - type = string + type = string } variable "aks_cluster_name" { - type = string + type = string } variable "tenant_id" { - type = string + type = string } variable "subscription_id" { - type = string + type = string } variable "hostname" { - type = string + type = string } variable "namespace" { - type = string + type = string } variable "service_account_name" { - type = string + type = string } variable "workload_managed_identity_client_id" { - type = string + type = string } variable "email" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf index 312190d28..aab17f34b 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf @@ -1,8 +1,9 @@ resource "azurerm_key_vault" "key_vault" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - tenant_id = var.tenant_id + name = var.name + location = var.location + resource_group_name = var.resource_group_name + tenant_id = var.tenant_id + sku_name = var.sku_name enabled_for_deployment = var.enabled_for_deployment enabled_for_disk_encryption = var.enabled_for_disk_encryption @@ -16,10 +17,8 @@ resource "azurerm_key_vault" "key_vault" { } network_acls { - bypass = var.bypass - default_action = var.default_action - ip_rules = var.ip_rules - virtual_network_subnet_ids = var.virtual_network_subnet_ids + bypass = var.bypass + default_action = var.default_action } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf index 3d727607e..ffb395cc4 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf @@ -1,9 +1,7 @@ output "name" { - value = azurerm_key_vault.key_vault.name - description = "Specifies the name of the key vault." + value = azurerm_key_vault.key_vault.name } output "id" { - value = azurerm_key_vault.key_vault.id - description = "Specifies the resource id of the key vault." + value = azurerm_key_vault.key_vault.id } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf index 628c6bdbc..3421eb126 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf @@ -1,115 +1,59 @@ variable "name" { - description = "(Required) Specifies the name of the key vault." - type = string + type = string } variable "resource_group_name" { - description = "(Required) Specifies the resource group name of the key vault." - type = string + type = string } variable "location" { - description = "(Required) Specifies the location where the key vault will be deployed." - type = string + type = string } variable "tenant_id" { - description = "(Required) The Azure Active Directory tenant ID that should be used for authenticating requests to the key vault." - type = string + type = string } variable "sku_name" { - description = "(Required) The Name of the SKU used for this Key Vault. Possible values are standard and premium." - type = string - default = "standard" - - validation { - condition = contains(["standard", "premium"], var.sku_name) - error_message = "The value of the sku name property of the key vault is invalid." - } -} - -variable "tags" { - description = "(Optional) Specifies the tags of the log analytics workspace" - type = map(any) - default = {} + type = string } variable "enabled_for_deployment" { - description = "(Optional) Boolean flag to specify whether Azure Virtual Machines are permitted to retrieve certificates stored as secrets from the key vault. Defaults to false." - type = bool - default = false + type = bool } variable "enabled_for_disk_encryption" { - description = " (Optional) Boolean flag to specify whether Azure Disk Encryption is permitted to retrieve secrets from the vault and unwrap keys. Defaults to false." - type = bool - default = false + type = bool } variable "enabled_for_template_deployment" { - description = "(Optional) Boolean flag to specify whether Azure Resource Manager is permitted to retrieve secrets from the key vault. Defaults to false." - type = bool - default = false + type = bool } variable "enable_rbac_authorization" { - description = "(Optional) Boolean flag to specify whether Azure Key Vault uses Role Based Access Control (RBAC) for authorization of data actions. Defaults to false." - type = bool - default = false + type = bool } variable "purge_protection_enabled" { - description = "(Optional) Is Purge Protection enabled for this Key Vault? Defaults to false." - type = bool - default = false + type = bool } variable "soft_delete_retention_days" { - description = "(Optional) The number of days that items should be retained for once soft-deleted. This value can be between 7 and 90 (the default) days." - type = number - default = 30 + type = number } variable "bypass" { - description = "(Required) Specifies which traffic can bypass the network rules. Possible values are AzureServices and None." - type = string - default = "AzureServices" - - validation { - condition = contains(["AzureServices", "None"], var.bypass) - error_message = "The valut of the bypass property of the key vault is invalid." - } + type = string } variable "default_action" { - description = "(Required) The Default Action to use when no rules match from ip_rules / virtual_network_subnet_ids. Possible values are Allow and Deny." - type = string - default = "Allow" - - validation { - condition = contains(["Allow", "Deny"], var.default_action) - error_message = "The value of the default action property of the key vault is invalid." - } -} - -variable "ip_rules" { - description = "(Optional) One or more IP Addresses, or CIDR Blocks which should be able to access the Key Vault." - default = [] -} - -variable "virtual_network_subnet_ids" { - description = "(Optional) One or more Subnet ID's which should be able to access this Key Vault." - default = [] + type = string } variable "log_analytics_workspace_id" { - description = "Specifies the log analytics workspace id" - type = string + type = string } variable "log_analytics_retention_days" { - description = "Specifies the number of days of the retention policy" - type = number - default = 7 + type = number } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf index 7e802cfe8..5f2bfe48d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf @@ -3,7 +3,7 @@ resource "azurerm_log_analytics_workspace" "log_analytics_workspace" { location = var.location resource_group_name = var.resource_group_name sku = var.sku - retention_in_days = var.retention_in_days != "" ? var.retention_in_days : null + retention_in_days = var.retention_in_days } resource "azurerm_log_analytics_solution" "la_solution" { @@ -19,10 +19,4 @@ resource "azurerm_log_analytics_solution" "la_solution" { product = each.value.product publisher = each.value.publisher } - - lifecycle { - ignore_changes = [ - tags - ] - } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf index ed214b0b1..6a0d04469 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf @@ -1,42 +1,23 @@ variable "resource_group_name" { - description = "(Required) Specifies the resource group name" - type = string + type = string } variable "location" { - description = "(Required) Specifies the location of the log analytics workspace" - type = string + type = string } variable "name" { - description = "(Required) Specifies the name of the log analytics workspace" - type = string + type = string } variable "sku" { - description = "(Optional) Specifies the sku of the log analytics workspace" - type = string - default = "PerGB2018" - - validation { - condition = contains(["Free", "Standalone", "PerNode", "PerGB2018"], var.sku) - error_message = "The log analytics sku is incorrect." - } + type = string } variable "solution_plan_map" { - description = "(Required) Specifies the map structure containing the list of solutions to be enabled." - type = map(any) -} - -variable "tags" { - description = "(Optional) Specifies the tags of the log analytics workspace" - type = map(any) - default = {} + type = map(any) } variable "retention_in_days" { - description = " (Optional) Specifies the workspace data retention in days. Possible values are either 7 (Free Tier only) or range between 30 and 730." - type = number - default = 30 + type = number } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf index 20b8af513..3b2964d0f 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf @@ -1,8 +1,8 @@ resource "azurerm_cognitive_account" "openai" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - + name = var.name + location = var.location + resource_group_name = var.resource_group_name + kind = "OpenAI" custom_subdomain_name = var.custom_subdomain_name sku_name = var.sku_name diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf index b687d39cd..86199689b 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf @@ -1,20 +1,14 @@ variable "name" { - description = "(Required) Specifies the name of the private dns zone" - type = string + type = string } variable "resource_group_name" { - description = "(Required) Specifies the resource group name of the private dns zone" - type = string -} - -variable "tags" { - description = "(Optional) Specifies the tags of the private dns zone" - default = {} + type = string } variable "virtual_networks_to_link" { - description = "(Optional) Specifies the subscription id, resource group name, and name of the virtual networks to which create a virtual network link" - type = map(any) - default = {} + type = map(string, object({ + subscription_id = string + resource_group_name = string + })) } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 5bb8b53d4..af24bc583 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -5,7 +5,7 @@ variable "resource_group_name_prefix" { variable "location" { type = string - default = "westus" + default = "westus3" } variable "kubernetes_version" { From fba749f1eb047dbff9e85c6872cbdf71c9d1b2e2 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Fri, 17 Jan 2025 12:47:33 -0800 Subject: [PATCH 045/119] WIP --- .../AksOpenAiTerraform/terraform/main.tf | 22 ++++++++++--------- .../modules/deployment_script/main.tf | 8 +++---- .../modules/deployment_script/variables.tf | 8 +++---- .../modules/private_dns_zone/variables.tf | 2 +- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index a2b4632e5..2e0b23b6d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -157,17 +157,19 @@ module "deployment_script" { location = var.location resource_group_name = azurerm_resource_group.rg.name - azure_cli_version = "2.64.0" - aks_cluster_id = module.aks_cluster.id + tenant_id = local.tenant_id + subscription_id = local.subscription_id + script_path = "./install-nginx-via-helm-and-create-sa.sh" + + azure_cli_version = "2.64.0" + aks_cluster_id = module.aks_cluster.id + aks_cluster_name = module.aks_cluster.name + hostname = "magic8ball.contoso.com" + namespace = local.namespace + service_account_name = local.service_account_name + email = var.email + managed_identity_name = "ScriptManagedIdentity" - aks_cluster_name = module.aks_cluster.name - hostname = "magic8ball.contoso.com" - namespace = local.namespace - service_account_name = local.service_account_name - email = var.email - primary_script_uri = "https://paolosalvatori.blob.core.windows.net/scripts/install-nginx-via-helm-and-create-sa.sh" - tenant_id = local.tenant_id - subscription_id = local.subscription_id workload_managed_identity_client_id = azurerm_user_assigned_identity.aks_workload_identity.client_id depends_on = [ diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf index 82d3368ee..3cce77dc7 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf @@ -15,13 +15,13 @@ resource "azurerm_resource_deployment_script_azure_cli" "script" { resource_group_name = var.resource_group_name location = var.location - version = var.azure_cli_version + version = var.azure_cli_version + script_content = file(var.script_path) + retention_interval = "P1D" - command_line = "'foo' 'bar'" cleanup_preference = "OnSuccess" - force_update_tag = "1" timeout = "PT30M" - primary_script_uri = var.primary_script_uri + force_update_tag = "1" identity { type = "UserAssigned" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf index 20ac1307e..332e60cea 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf @@ -10,19 +10,19 @@ variable "name" { type = string } -variable "aks_cluster_id" { +variable "script_path" { type = string } -variable "azure_cli_version" { +variable "aks_cluster_id" { type = string } -variable "managed_identity_name" { +variable "azure_cli_version" { type = string } -variable "primary_script_uri" { +variable "managed_identity_name" { type = string } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf index 86199689b..ce748b6f9 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf @@ -7,7 +7,7 @@ variable "resource_group_name" { } variable "virtual_networks_to_link" { - type = map(string, object({ + type = map(object({ subscription_id = string resource_group_name = string })) From 89ac91156b323a2b73828a808e203a7141808210 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Fri, 17 Jan 2025 13:17:04 -0800 Subject: [PATCH 046/119] Remove deploy --- .../AksOpenAiTerraform/terraform/main.tf | 42 ++-------- .../modules/deployment_script/main.tf | 82 ------------------- .../modules/deployment_script/variables.tf | 60 -------------- 3 files changed, 8 insertions(+), 176 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 2e0b23b6d..c62f844a1 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -22,6 +22,14 @@ resource "random_string" "rg_suffix" { numeric = true } +resource "random_string" "storage_account_suffix" { + length = 8 + special = false + lower = true + upper = false + numeric = false +} + locals { tenant_id = data.azurerm_client_config.current.tenant_id subscription_id = data.azurerm_client_config.current.subscription_id @@ -39,14 +47,6 @@ locals { log_analytics_retention_days = 30 } -resource "random_string" "storage_account_suffix" { - length = 8 - special = false - lower = true - upper = false - numeric = false -} - resource "azurerm_resource_group" "rg" { name = "${var.resource_group_name_prefix}-${local.random_id}-rg" location = var.location @@ -151,32 +151,6 @@ module "key_vault" { log_analytics_retention_days = local.log_analytics_retention_days } -module "deployment_script" { - source = "./modules/deployment_script" - name = "DeployBashScript" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - - tenant_id = local.tenant_id - subscription_id = local.subscription_id - script_path = "./install-nginx-via-helm-and-create-sa.sh" - - azure_cli_version = "2.64.0" - aks_cluster_id = module.aks_cluster.id - aks_cluster_name = module.aks_cluster.name - hostname = "magic8ball.contoso.com" - namespace = local.namespace - service_account_name = local.service_account_name - email = var.email - - managed_identity_name = "ScriptManagedIdentity" - workload_managed_identity_client_id = azurerm_user_assigned_identity.aks_workload_identity.client_id - - depends_on = [ - module.aks_cluster - ] -} - module "log_analytics_workspace" { source = "./modules/log_analytics" name = local.log_analytics_workspace_name diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf deleted file mode 100644 index 3cce77dc7..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/main.tf +++ /dev/null @@ -1,82 +0,0 @@ -resource "azurerm_user_assigned_identity" "script_identity" { - name = var.managed_identity_name - location = var.location - resource_group_name = var.resource_group_name -} - -resource "azurerm_role_assignment" "network_contributor_assignment" { - scope = var.aks_cluster_id - role_definition_name = "Azure Kubernetes Service Cluster Admin Role" - principal_id = azurerm_user_assigned_identity.script_identity.principal_id -} - -resource "azurerm_resource_deployment_script_azure_cli" "script" { - name = var.name - resource_group_name = var.resource_group_name - location = var.location - - version = var.azure_cli_version - script_content = file(var.script_path) - - retention_interval = "P1D" - cleanup_preference = "OnSuccess" - timeout = "PT30M" - force_update_tag = "1" - - identity { - type = "UserAssigned" - identity_ids = [ - azurerm_user_assigned_identity.script_identity.id - ] - } - - environment_variable { - name = "clusterName" - value = var.aks_cluster_name - } - - environment_variable { - name = "resourceGroupName" - value = var.resource_group_name - } - - environment_variable { - name = "applicationGatewayEnabled" - value = false - } - - environment_variable { - name = "tenantId" - value = var.tenant_id - } - - environment_variable { - name = "subscriptionId" - value = var.subscription_id - } - - environment_variable { - name = "hostName" - value = var.hostname - } - - environment_variable { - name = "namespace" - value = var.namespace - } - - environment_variable { - name = "serviceAccountName" - value = var.service_account_name - } - - environment_variable { - name = "workloadManagedIdentityClientId" - value = var.workload_managed_identity_client_id - } - - environment_variable { - name = "email" - value = var.email - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf deleted file mode 100644 index 332e60cea..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/deployment_script/variables.tf +++ /dev/null @@ -1,60 +0,0 @@ -variable "resource_group_name" { - type = string -} - -variable "location" { - type = string -} - -variable "name" { - type = string -} - -variable "script_path" { - type = string -} - -variable "aks_cluster_id" { - type = string -} - -variable "azure_cli_version" { - type = string -} - -variable "managed_identity_name" { - type = string -} - -variable "aks_cluster_name" { - type = string -} - -variable "tenant_id" { - type = string -} - -variable "subscription_id" { - type = string -} - -variable "hostname" { - type = string -} - -variable "namespace" { - type = string -} - -variable "service_account_name" { - type = string -} - -variable "workload_managed_identity_client_id" { - type = string -} - -variable "email" { - description = "Specifies the email address for the cert-manager cluster issuer." - type = string -} \ No newline at end of file From 966d7583929ac578f4759339be54d3f6570d7c24 Mon Sep 17 00:00:00 2001 From: "Aria Amini (from Dev Box)" Date: Fri, 17 Jan 2025 13:17:22 -0800 Subject: [PATCH 047/119] New directory structure --- scenarios/AksOpenAiTerraform/run.sh | 73 ++++ .../AksOpenAiTerraform/script/app/Dockerfile | 94 +++++ .../AksOpenAiTerraform/script/app/app.py | 347 ++++++++++++++++++ .../script/app/images/magic8ball.png | Bin 0 -> 37452 bytes .../script/app/images/robot.png | Bin 0 -> 1686 bytes .../script/app/requirements.txt | 145 ++++++++ .../install-nginx-via-helm-and-create-sa.sh | 0 .../script/manifests/cluster-issuer.yml | 18 + .../script/manifests/configMap.yml | 14 + .../script/manifests/deployment.yml | 123 +++++++ .../script/manifests/ingress.yml | 30 ++ .../script/manifests/service.yml | 13 + 12 files changed, 857 insertions(+) create mode 100644 scenarios/AksOpenAiTerraform/run.sh create mode 100644 scenarios/AksOpenAiTerraform/script/app/Dockerfile create mode 100644 scenarios/AksOpenAiTerraform/script/app/app.py create mode 100644 scenarios/AksOpenAiTerraform/script/app/images/magic8ball.png create mode 100644 scenarios/AksOpenAiTerraform/script/app/images/robot.png create mode 100644 scenarios/AksOpenAiTerraform/script/app/requirements.txt rename scenarios/AksOpenAiTerraform/{terraform => script}/install-nginx-via-helm-and-create-sa.sh (100%) create mode 100644 scenarios/AksOpenAiTerraform/script/manifests/cluster-issuer.yml create mode 100644 scenarios/AksOpenAiTerraform/script/manifests/configMap.yml create mode 100644 scenarios/AksOpenAiTerraform/script/manifests/deployment.yml create mode 100644 scenarios/AksOpenAiTerraform/script/manifests/ingress.yml create mode 100644 scenarios/AksOpenAiTerraform/script/manifests/service.yml diff --git a/scenarios/AksOpenAiTerraform/run.sh b/scenarios/AksOpenAiTerraform/run.sh new file mode 100644 index 000000000..adebad18e --- /dev/null +++ b/scenarios/AksOpenAiTerraform/run.sh @@ -0,0 +1,73 @@ +export OPEN_AI_SUBDOMAIN="magic8ball" + +# Variables +acrName="CyanAcr" +acrResourceGrougName="CyanRG" +location="FranceCentral" +attachAcr=false +imageName="magic8ball" +tag="v2" +containerName="magic8ball" +image="$acrName.azurecr.io/$imageName:$tag" +imagePullPolicy="IfNotPresent" # Always, Never, IfNotPresent +managedIdentityName="CyanWorkloadManagedIdentity" +federatedIdentityName="Magic8BallFederatedIdentity" + +# Azure Subscription and Tenant +subscriptionId=$(az account show --query id --output tsv) +subscriptionName=$(az account show --query name --output tsv) +tenantId=$(az account show --query tenantId --output tsv) + +# Parameters +title="Magic 8 Ball" +label="Pose your question and cross your fingers!" +temperature="0.9" +imageWidth="80" + +# OpenAI +openAiName="CyanOpenAi " +openAiResourceGroupName="CyanRG" +openAiType="azure_ad" +openAiBase="https://cyanopenai.openai.azure.com/" +openAiModel="gpt-35-turbo" +openAiDeployment="gpt-35-turbo" + +# Nginx Ingress Controller +nginxNamespace="ingress-basic" +nginxRepoName="ingress-nginx" +nginxRepoUrl="https://kubernetes.github.io/ingress-nginx" +nginxChartName="ingress-nginx" +nginxReleaseName="nginx-ingress" +nginxReplicaCount=3 + +# Certificate Manager +cmNamespace="cert-manager" +cmRepoName="jetstack" +cmRepoUrl="https://charts.jetstack.io" +cmChartName="cert-manager" +cmReleaseName="cert-manager" + +# Cluster Issuer +email="paolos@microsoft.com" +clusterIssuerName="letsencrypt-nginx" +clusterIssuerTemplate="cluster-issuer.yml" + +# AKS Cluster +aksClusterName="CyanAks" +aksResourceGroupName="CyanRG" + +# Sample Application +namespace="magic8ball" +serviceAccountName="magic8ball-sa" +deploymentTemplate="deployment.yml" +serviceTemplate="service.yml" +configMapTemplate="configMap.yml" +secretTemplate="secret.yml" + +# Ingress and DNS +ingressTemplate="ingress.yml" +ingressName="magic8ball-ingress" +dnsZoneName="contoso.com" +dnsZoneResourceGroupName="DnsResourceGroup" +subdomain="magic" +host="$subdomain.$dnsZoneName" \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/script/app/Dockerfile b/scenarios/AksOpenAiTerraform/script/app/Dockerfile new file mode 100644 index 000000000..2f603014f --- /dev/null +++ b/scenarios/AksOpenAiTerraform/script/app/Dockerfile @@ -0,0 +1,94 @@ +# app/Dockerfile + +# # Stage 1 - Install build dependencies + +# A Dockerfile must start with a FROM instruction which sets the base image for the container. +# The Python images come in many flavors, each designed for a specific use case. +# The python:3.11-slim image is a good base image for most applications. +# It is a minimal image built on top of Debian Linux and includes only the necessary packages to run Python. +# The slim image is a good choice because it is small and contains only the packages needed to run Python. +# For more information, see: +# * https://hub.docker.com/_/python +# * https://docs.streamlit.io/knowledge-base/tutorials/deploy/docker +FROM python:3.11-slim AS builder + +# The WORKDIR instruction sets the working directory for any RUN, CMD, ENTRYPOINT, COPY and ADD instructions that follow it in the Dockerfile. +# If the WORKDIR doesn’t exist, it will be created even if it’s not used in any subsequent Dockerfile instruction. +# For more information, see: https://docs.docker.com/engine/reference/builder/#workdir +WORKDIR /app + +# Set environment variables. +# The ENV instruction sets the environment variable to the value . +# This value will be in the environment of all “descendant” Dockerfile commands and can be replaced inline in many as well. +# For more information, see: https://docs.docker.com/engine/reference/builder/#env +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# Install git so that we can clone the app code from a remote repo using the RUN instruction. +# The RUN comand has 2 forms: +# * RUN (shell form, the command is run in a shell, which by default is /bin/sh -c on Linux or cmd /S /C on Windows) +# * RUN ["executable", "param1", "param2"] (exec form) +# The RUN instruction will execute any commands in a new layer on top of the current image and commit the results. +# The resulting committed image will be used for the next step in the Dockerfile. +# For more information, see: https://docs.docker.com/engine/reference/builder/#run +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + software-properties-common \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Create a virtualenv to keep dependencies together +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +# Clone the requirements.txt which contains dependencies to WORKDIR +# COPY has two forms: +# * COPY (this copies the files from the local machine to the container's own filesystem) +# * COPY ["",... ""] (this form is required for paths containing whitespace) +# For more information, see: https://docs.docker.com/engine/reference/builder/#copy +COPY requirements.txt . + +# Install the Python dependencies +RUN pip install --no-cache-dir --no-deps -r requirements.txt + +# Stage 2 - Copy only necessary files to the runner stage + +# The FROM instruction initializes a new build stage for the application +FROM python:3.11-slim + +# Sets the working directory to /app +WORKDIR /app + +# Copy the virtual environment from the builder stage +COPY --from=builder /opt/venv /opt/venv + +# Set environment variables +ENV PATH="/opt/venv/bin:$PATH" + +# Clone the app.py containing the application code +COPY app.py . + +# Copy the images folder to WORKDIR +# The ADD instruction copies new files, directories or remote file URLs from and adds them to the filesystem of the image at the path . +# For more information, see: https://docs.docker.com/engine/reference/builder/#add +ADD images ./images + +# The EXPOSE instruction informs Docker that the container listens on the specified network ports at runtime. +# For more information, see: https://docs.docker.com/engine/reference/builder/#expose +EXPOSE 8501 + +# The HEALTHCHECK instruction has two forms: +# * HEALTHCHECK [OPTIONS] CMD command (check container health by running a command inside the container) +# * HEALTHCHECK NONE (disable any healthcheck inherited from the base image) +# The HEALTHCHECK instruction tells Docker how to test a container to check that it is still working. +# This can detect cases such as a web server that is stuck in an infinite loop and unable to handle new connections, +# even though the server process is still running. For more information, see: https://docs.docker.com/engine/reference/builder/#healthcheck +HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health + +# The ENTRYPOINT instruction has two forms: +# * ENTRYPOINT ["executable", "param1", "param2"] (exec form, preferred) +# * ENTRYPOINT command param1 param2 (shell form) +# The ENTRYPOINT instruction allows you to configure a container that will run as an executable. +# For more information, see: https://docs.docker.com/engine/reference/builder/#entrypoint +ENTRYPOINT ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"] \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/script/app/app.py b/scenarios/AksOpenAiTerraform/script/app/app.py new file mode 100644 index 000000000..4211c57ca --- /dev/null +++ b/scenarios/AksOpenAiTerraform/script/app/app.py @@ -0,0 +1,347 @@ +""" +MIT License + +Copyright (c) 2023 Paolo Salvatori + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +# This sample is based on the following article: +# +# - https://levelup.gitconnected.com/its-time-to-create-a-private-chatgpt-for-yourself-today-6503649e7bb6 +# +# Use pip to install the following packages: +# +# - streamlit +# - openai +# - streamlit-chat +# - azure.identity +# - dotenv +# +# Make sure to provide a value for the following environment variables: +# +# - AZURE_OPENAI_BASE: the URL of your Azure OpenAI resource, for example https://eastus.api.cognitive.microsoft.com/ +# - AZURE_OPENAI_KEY: the key of your Azure OpenAI resource +# - AZURE_OPENAI_DEPLOYMENT: the name of the ChatGPT deployment used by your Azure OpenAI resource +# - AZURE_OPENAI_MODEL: the name of the ChatGPT model used by your Azure OpenAI resource, for example gpt-35-turbo +# - TITLE: the title of the Streamlit app +# - TEMPERATURE: the temperature used by the OpenAI API to generate the response +# - SYSTEM: give the model instructions about how it should behave and any context it should reference when generating a response. +# Used to describe the assistant's personality. +# +# You can use two different authentication methods: +# +# - API key: set the AZURE_OPENAI_TYPE environment variable to azure and the AZURE_OPENAI_KEY environment variable to the key of +# your Azure OpenAI resource. You can use the regional endpoint, such as https://eastus.api.cognitive.microsoft.com/, passed in +# the AZURE_OPENAI_BASE environment variable, to connect to the Azure OpenAI resource. +# - Azure Active Directory: set the AZURE_OPENAI_TYPE environment variable to azure_ad and use a service principal or managed +# identity with the DefaultAzureCredential object to acquire a token. For more information on the DefaultAzureCredential in Python, +# see https://docs.microsoft.com/en-us/azure/developer/python/azure-sdk-authenticate?tabs=cmd +# Make sure to assign the "Cognitive Services User" role to the service principal or managed identity used to authenticate to +# Azure OpenAI. For more information, see https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/managed-identity. +# If you want to use Azure AD integrated security, you need to create a custom subdomain for your Azure OpenAI resource and use the +# specific endpoint containing the custom domain, such as https://bingo.openai.azure.com/ where bingo is the custom subdomain. +# If you specify the regional endpoint, you get a wonderful error: "Subdomain does not map to a resource.". +# Hence, make sure to pass the endpoint containing the custom domain in the AZURE_OPENAI_BASE environment variable. +# +# Use the following command to run the app: +# +# - streamlit run app.py + +# Import packages +import os +import sys +import time +import openai +import logging +import streamlit as st +from streamlit_chat import message +from azure.identity import DefaultAzureCredential +from dotenv import load_dotenv +from dotenv import dotenv_values + +# Load environment variables from .env file +if os.path.exists(".env"): + load_dotenv(override=True) + config = dotenv_values(".env") + +# Read environment variables +assistan_profile = """ +You are the infamous Magic 8 Ball. You need to randomly reply to any question with one of the following answers: + +- It is certain. +- It is decidedly so. +- Without a doubt. +- Yes definitely. +- You may rely on it. +- As I see it, yes. +- Most likely. +- Outlook good. +- Yes. +- Signs point to yes. +- Reply hazy, try again. +- Ask again later. +- Better not tell you now. +- Cannot predict now. +- Concentrate and ask again. +- Don't count on it. +- My reply is no. +- My sources say no. +- Outlook not so good. +- Very doubtful. + +Add a short comment in a pirate style at the end! Follow your heart and be creative! +For mor information, see https://en.wikipedia.org/wiki/Magic_8_Ball +""" +title = os.environ.get("TITLE", "Magic 8 Ball") +text_input_label = os.environ.get("TEXT_INPUT_LABEL", "Pose your question and cross your fingers!") +image_file_name = os.environ.get("IMAGE_FILE_NAME", "magic8ball.png") +image_width = int(os.environ.get("IMAGE_WIDTH", 80)) +temperature = float(os.environ.get("TEMPERATURE", 0.9)) +system = os.environ.get("SYSTEM", assistan_profile) +api_base = os.getenv("AZURE_OPENAI_BASE") +api_key = os.getenv("AZURE_OPENAI_KEY") +api_type = os.environ.get("AZURE_OPENAI_TYPE", "azure") +api_version = os.environ.get("AZURE_OPENAI_VERSION", "2023-05-15") +engine = os.getenv("AZURE_OPENAI_DEPLOYMENT") +model = os.getenv("AZURE_OPENAI_MODEL") + +# Configure OpenAI +openai.api_type = api_type +openai.api_version = api_version +openai.api_base = api_base + +# Set default Azure credential +default_credential = DefaultAzureCredential() if openai.api_type == "azure_ad" else None + +# Configure a logger +logging.basicConfig(stream = sys.stdout, + format = '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s', + level = logging.INFO) +logger = logging.getLogger(__name__) + +# Log variables +logger.info(f"title: {title}") +logger.info(f"text_input_label: {text_input_label}") +logger.info(f"image_file_name: {image_file_name}") +logger.info(f"image_width: {image_width}") +logger.info(f"temperature: {temperature}") +logger.info(f"system: {system}") +logger.info(f"api_base: {api_base}") +logger.info(f"api_key: {api_key}") +logger.info(f"api_type: {api_type}") +logger.info(f"api_version: {api_version}") +logger.info(f"engine: {engine}") +logger.info(f"model: {model}") + +# Authenticate to Azure OpenAI +if openai.api_type == "azure": + openai.api_key = api_key +elif openai.api_type == "azure_ad": + openai_token = default_credential.get_token("https://cognitiveservices.azure.com/.default") + openai.api_key = openai_token.token + if 'openai_token' not in st.session_state: + st.session_state['openai_token'] = openai_token +else: + logger.error("Invalid API type. Please set the AZURE_OPENAI_TYPE environment variable to azure or azure_ad.") + raise ValueError("Invalid API type. Please set the AZURE_OPENAI_TYPE environment variable to azure or azure_ad.") + +# Customize Streamlit UI using CSS +st.markdown(""" + +""", unsafe_allow_html=True) + +# Initialize Streamlit session state +if 'prompts' not in st.session_state: + st.session_state['prompts'] = [{"role": "system", "content": system}] + +if 'generated' not in st.session_state: + st.session_state['generated'] = [] + +if 'past' not in st.session_state: + st.session_state['past'] = [] + +# Refresh the OpenAI security token every 45 minutes +def refresh_openai_token(): + if st.session_state['openai_token'].expires_on < int(time.time()) - 45 * 60: + st.session_state['openai_token'] = default_credential.get_token("https://cognitiveservices.azure.com/.default") + openai.api_key = st.session_state['openai_token'].token + +# Send user prompt to Azure OpenAI +def generate_response(prompt): + try: + st.session_state['prompts'].append({"role": "user", "content": prompt}) + + if openai.api_type == "azure_ad": + refresh_openai_token() + + completion = openai.ChatCompletion.create( + engine = engine, + model = model, + messages = st.session_state['prompts'], + temperature = temperature, + ) + + message = completion.choices[0].message.content + return message + except Exception as e: + logging.exception(f"Exception in generate_response: {e}") + +# Reset Streamlit session state to start a new chat from scratch +def new_click(): + st.session_state['prompts'] = [{"role": "system", "content": system}] + st.session_state['past'] = [] + st.session_state['generated'] = [] + st.session_state['user'] = "" + +# Handle on_change event for user input +def user_change(): + # Avoid handling the event twice when clicking the Send button + chat_input = st.session_state['user'] + st.session_state['user'] = "" + if (chat_input == '' or + (len(st.session_state['past']) > 0 and chat_input == st.session_state['past'][-1])): + return + + # Generate response invoking Azure OpenAI LLM + if chat_input != '': + output = generate_response(chat_input) + + # store the output + st.session_state['past'].append(chat_input) + st.session_state['generated'].append(output) + st.session_state['prompts'].append({"role": "assistant", "content": output}) + +# Create a 2-column layout. Note: Streamlit columns do not properly render on mobile devices. +# For more information, see https://github.com/streamlit/streamlit/issues/5003 +col1, col2 = st.columns([1, 7]) + +# Display the robot image +with col1: + st.image(image = os.path.join("images", image_file_name), width = image_width) + +# Display the title +with col2: + st.title(title) + +# Create a 3-column layout. Note: Streamlit columns do not properly render on mobile devices. +# For more information, see https://github.com/streamlit/streamlit/issues/5003 +col3, col4, col5 = st.columns([7, 1, 1]) + +# Create text input in column 1 +with col3: + user_input = st.text_input(text_input_label, key = "user", on_change = user_change) + +# Create send button in column 2 +with col4: + st.button(label = "Send") + +# Create new button in column 3 +with col5: + st.button(label = "New", on_click = new_click) + +# Display the chat history in two separate tabs +# - normal: display the chat history as a list of messages using the streamlit_chat message() function +# - rich: display the chat history as a list of messages using the Streamlit markdown() function +if st.session_state['generated']: + tab1, tab2 = st.tabs(["normal", "rich"]) + with tab1: + for i in range(len(st.session_state['generated']) - 1, -1, -1): + message(st.session_state['past'][i], is_user = True, key = str(i) + '_user', avatar_style = "fun-emoji", seed = "Nala") + message(st.session_state['generated'][i], key = str(i), avatar_style = "bottts", seed = "Fluffy") + with tab2: + for i in range(len(st.session_state['generated']) - 1, -1, -1): + st.markdown(st.session_state['past'][i]) + st.markdown(st.session_state['generated'][i]) \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/script/app/images/magic8ball.png b/scenarios/AksOpenAiTerraform/script/app/images/magic8ball.png new file mode 100644 index 0000000000000000000000000000000000000000..cd53753774ed4e666c7093f6d58ca02a25be36a1 GIT binary patch literal 37452 zcmW(+1yEaUv&IP?+#$HTI|O$v?ozC1ahKrkTHIRPiWDzyrC5s=_YW@)z30D~$s}`T zk~x#?yZhM2X=x~8p_8G*!NFlEE6M4=!NIft?+HQxuG~?R(gH8+Uh;-svYvKUUXE_A z5Isj1J2(YTM+mPVMBU0C!pqIe2jLbG<>MFS=Yg|VxholcgoDGt``;6Oek0@sxQJvgqb>sn*PMd!VucI`msYMUC!^=LdKUcn zG3%HAzpmfP6w;t@`nG0JGYB4G`n_15u+NT9e&GQEGi-!PDoUaMy*K!6^#-|!cBFAl#g zs~yWDLx!VYXUHQY#NmJ_eHYN0BxA^V0Iw}hJ{wqC44>j`d?-vW2Qb~WRivrLO z$;rsb$jKk;APW+HI@$mG79HZjM2I~rNB{H21(mp8*q=XdcO9V*->Kg+iL7yug@<3^ z(%JC-=0Ww5QeTj`AG>}8pWhV!dm)~0;bjvxFDAm$UhiN3bi2Z;DQ$R@Zi_rD!N-T# z6MQ|vG_ZCN`Y#ZElY0KTx{loz`S&jTpYJv@lvh>#I+uj;as*Vn#ACx1kedXHmmR(z z;*o!rY7IK7kf{SACMsE?m~;u+nGwjfE`R_2{qXR>)mJ)& zoK8$k93MM|Vby#K{r5KJSbBuNfj2Tl5=97+fgiCUC?ZCSMr>4wucc*`dWXz(B@g{% zzehz|?8f*jH3hGbAhA}z9)aNW^fY{;s)`Y$yp21xv7e&sk1#YslrG79yak6#Vzvc#c(E}qLB!8{nYtm^ zcxh;0J&9gLh{wXh;)eWcF#BfpJvuoVyG%Rr2UBGWP!WVgx8yK1>=G9ZH8r(rK~w4nK zFp(8>Dw%`QkICGgCWu-8K_~ zexOksPIg3c)3E=zg5g)+1!xZ5?3Vb_`#d{%UK;Y1%OrB4^OaWAoXaAnq9t*l6l~;& zF}yT#g9bLIr!g-Xvk9w-q$5hCln_%P!w6xhxbSC=^s)X`Vy07^GF5&mZD!R>NA0&C zlVp?Be9^Tp9W%|H{3Z8g{zDy-a1a=#n^U4elP*i@Q4}+{N}vi;I#Wqd-ao|WSD3^H zKVc|;3<@gzT~JVfc;|58AxZg)QjVSKtfXpVLzgTQ%gey*lQT|Bc>&$FiT`dfJVxix z>^`NKI22bDlR9aV@3fcR_a)0v)p(NON$=RT@|ienn5!H4K8zF|R;-GV{Fwj_`2S44 zekeV89(M4W!UwNLk?kHGF;`HlNybXft+*{9=pd4YFW`qjxRj&}1NbxG#6@Y~F|`lt zy2P-QFtj%<`GZ-!z`S(o5VBcks}u4%zOpg&5awlOa(MYBrr8MIld#by!+z$b7%LvA z$e5e6GaJ+U?7%|mGN2^?8H^vet#^9d9L!8iC>sVt*m?FG9_P?xc2QM`XLHq@2~%n& zl=Kw?yGLLic4d&;zPX1*?XDGJ+FuDm1~n}p?E0HUzMoD5DSTxQ`-#le6pt7bP}cb< z#Lpm#uN@r=G;QlPZ(5m5kQWP(Oi*|_3My($baY9??2jJ{h=Vu7RKZsd$QIRsS`lB` zZ)DrFwB^*{v|j?0AEKxxQtZvH$0rIuDa0||Q57U&%lz8t`WO>7YynA<`O-!OFJE&$ zu$|o|&Ov!_uu-ns)%d|sBIEI{SbWZYcB&KlgkQl5vIkXP0i1lepmCA`l#q}1{ z3k1D^hnLqJ^|$r5rVsnA7|`fA)4m{51aLmnD`#d1|4E5^gj_&CfO~31ACYu&2W~O% z$5dNw7G4=mX=BfMc|AF1qa1dmlfb?7sD{{*)}}8%0zI5JJ{11;Frz;C55}yg+b|ko z37b?=e(=&C*b=V%xv_yI9rC`KUoruLBmn1#h=>>*9Q^(J_Y#~*AqdzDkrm!;^Q#N_ z8Icy+JN9(q6vx9Y3C!noG|{{#d?8B^%-nD^s^b2O0j0o*Qmgd&NYGx?IV)EA@<-6O zpUu2y%rE=XlHCZD$bv#VJn;Nwnk1CTwJ+es=exGB{q3CXz$tY)RbxpLh^^f3$D2c- zP6qQh^EqMYHSnhXK*h{u%FQ{hj0g+-9JcK(hUt^H=MM_2l0ro*u4I@iGF9q(xP7RF z;6G}I{KZg*yHN<&DoW4DhBiE_E7lZbzQ#=?@!ggncr*UJ8qh*|K0P~w)j#CvXi2d_ zY`5NphDvP-2?%)KpKtuBMz#&Fg+$FbmTr$hNN~dfoYrhL^bm9w3bl%|Ll`@X7asPq zU|)~F8X9DTt&$GCqln=|s7FoGgkAs6*W)+({RO*Zgo{8Aq|hXViT=i0LByN!m6qLo zTy_hL&Pr*S672sXqvY@X->QJdD|`rEOX63=kgYxvw?qjg_!q;SDcJDH$Pkp2WPU#p zu_zVh;XxAaP=?fk#@xnO7o?!4dpam=+N;@Yh)-f}tD~si0($3Aof=O<*}!@*i8pIs zXVD)rmQ2~(+l%>R;!69)#eZ4`RDo5E3<8Rk@udU<-hjkWbP|3jIvRy%o1JZzS$>gi z!PGYvC6O(8PeWJ5`&QEKxMO*(hl>Y2X45zNe|PKe%OjFv40 zMaj6po2e}(bAw3-_A-3p$3)y&sZp(s8Bc`vT5=D2V6N(csn!$Ut(9wr5r}+9AdUic^k#s@xQ2h6# zoco=*IkbYM1z#dRp}m3LA!}uF>!Bd-V?`lQ?zg`hgpmvkR`29u1*a% z_DG;a!zfL_#j~t;vT3{&xhPk3v4=Esb33D1qmox#mP3mX7A7-6-sjIOi8=JUrFuw= zR&(SI%T4_Auk&! zI+35@?b$~iQ#C_^aphetrwN^gt?T9CCwE$gflr>CdSHooPYR_7F;5Wl1H2U~MP;80Oeju0x#k!4lk zK@TqLow&QhhNP0#QIrLXWc;Kq{w_mcgi6@aQf9i;lLT-ojy5HZCHMO(*pU=Y+}cIo z>MMj_)TxzT5dV1+;X3RkTI=fKYmLq65{!oKq8O7tL`|ZLg#vxdRQE16Tag4tf(W@h zTWOVs+ywDXBOeaR!S~mjbO;Ix>L6#Gb<>`<`RrkFX5d8a6?sQ`90^p2<_B4i!Ahrc z{N&s2joiw2mbfLiqTaIWqC5V0DRjxjDHb8|cHKqv;6W-PlMYF_WXch>V5LeItL27N z@4#jEjb`6aNR^10bveL}00sz0UtL;i2M%V#o5rFsDyg5_d3G%SB~=0Qd?F6o=8uu$ zc)bY>zWbHAmN@Y+mJY|Xdy^4ldPI*T-e2%iAX#qR7tc~sVJ_K$yuD<|@I^QxD)-Km zdTv&xA|A{#{_saJD#&f4J1T7;OlA!UWVH$(qL@PE;_5mIAr&~awBv~nn^Zl2a?;@C zzL^u9Zw7fPUqC zw(>`xsz^ANf6=)$X&84G_YM9>>W05JP>W)6fwLsl=J2~7uU%J}00CPy4W#Dz%Ea-H z-eekWALR7&?e#ej7`BI=3y!4w+6Xkn&cz}hQ4}D=zf|7Skc0!hHU@_p1tAYXPcU%K z=oQZbE(ng4^54PgnG1x;ns22a=@vRxbUgs8F~@F=Vx0urRX)&i>m(t zYvE+3ljRXOYYgBWh{};OU)kRB2$ZZpoYJw%u+3~Y^Y$j6t)iyoT zyeZX18N=;tEc#*aS& zNe1V3!ejlI#vZ#95fxkdgX9b24jmknWXJwT5hVqM?VU49d1)Bkxh4hyj*3(w)-UCw zAp9Yf40j%F@7cYkNi_mKMpTZ2Al6y-L>$Ms-B}MYks%q{zxU?x_qH{#d^+<5`kC4k zi=jl3PVU=bnpx?m?Hp_okn(!;4Y!V{m>Bi-N4=Uq=1>ii^S#_8PI^}M z=p|yu7QZpkk~c#7u`DOCZnyAv!ur;$Y0<9(qoPxnepV2o-`<~zx)k)Zz$ z>iRoW%s*t-RdnQSqJQJ+pt?0j56-^jt9FpYy`p4DH)>QD7@+gZT>lR+2 z2G7a7=|m2DbmvSNE{UUYMJSZVql&;a2`A8XW<#~K;iHLF{Ee$NRf9!86)WB}QCKP~ z&aRuK1hmo`Bz#7kI;Vr{xp_So4wmqf;3zLI_k!WUj^?VWs_ct{?*_sVk38;JsFe$? zKM85m9Q3YKiU={}p)F(w3LsRsq!N7&eXE*~>H1k+>ej{N?e@)rOGG*?+@`q&=cB?2 zAVSc}(fT4Y5umA#x|w!+#c?jgxSZ^4BW%Au|F6p~|N71WpZQ+0d$-=m-9 zbI@Y{1JZTI6!*5Cg24GXW~FpaDgA^7PTh0BhMrebv@>|mJ=jgq_b-Oe8z9PBgR{bR$y1^x7xgOcLL z+XPr&ChTTsXUoQ5d;9u+)J`w5_$qjmu*(e`60O!Mb+rCG0K zjw-e4vJ3Y9h58#x$G%0Nqmzyw`zg;Lwc~dW4@RQ;UTk6^@tTs0qLeF`c?!WZeA(kZ z>>?<$#Fw^I2Iq56C%QS-^{^f?=$@HkO~~R$sZ(gX{^&%+?UwSIs5d`~!n2s9jItY) z+@tA5F|}!ST4F7-!ep&(?7OYxlJPZj9s2j@S}P2Eb~2ho4m3qdDysL)n9@o-v3-$5 z=&$gv<96XkG7}x|;6du2t$eurJ9t`51zdv-p+8N|4^`%SuS$Qpw4=eX&)-dgNj}?F z@X5NxN~PEA(Q&?xPd8DqmnDBz_s>c!;Cr*Y>V)rJipC<(&bs^8Ax;3Hprnk#q#z?J zV~UPv9)>)fj;GNzRSY&Gg)?`m%;r$Tp1z=^>b>abh@%lbWxuO@5Tko4y&9Z;HoH$> z(s5gx*%>&luf9)ME>I_MEPY}cdEeh-vCgNKG$>AdW;pr%>FJ4nN!5c&59!3(5=$fG zVLK)@=7;zV=DCr^Nhpal2&qWe!#X6S=<(+qR~Yp>kVEO#JGqjx?}fJTE2C+!IL%{T zY-GO#fvK~ZzXU5krlYKn(nH=clk0@|?e!Bt{gk;$sJZbhma~$ zs%@>c^TBe;y(sTz@}kn`zMbib3?`4Y?e8%;23%(E!J9q)H@C+Ne0+SZt*z_Dm}aBx zxMpT%O+|oOL4)IHSP=FZzq)DMA_GyIQ91(GnCm_r&Cj8S(5E2pMyKL!b9JvDQ|i7+q&$W0e?V8$4NoVGY<9X zY8+jbIQ#rr)%<$-kDPx}VrIu%>DsNuy$)zJpBuWn-)qYeycjjn^Q+ECmbL7A3O4c! z@&YP^iIFk44JG==d}^;rHWqAmcbAyU9L6~rg$qY@vsCW;)$I>9R%vT%#zFCiP&P=L z(j-iAiY(BVuy~P)NhVOLo-1necv+YD^lDQ2NPjdFaVY%{>&mt{cjgHDWbovg&!vI*9ZXK@Dm4w?&(ylGQvIk*(rsceF49AilRrqRsK}5@{YIz8(*T%j> z5_0|`6Fg`iVd+YI+DHKwgfb1*5iS+TOCEj*^tEBIZHIMpRr#P^X1$B885 ze zGpa=_KOFA_iz8Hn-qXqzwq7#|&F=stq$y5Mcs{Q*WAhyA9lIp+X7iLT+f}}&gwTJo zB;h5r?m^5Jn9m0hmhPd2UH&(W_u)=1E@mon>;fPfxJJ`11ZP8^+U!Dt|Ni^$`Tj!D z7cObugV6-S6<2VJSxWU4)P4-VFm2fKsI70Wc48FRoDHslcDikw=0Bq3(O=s;y1=TXG%RJ=8&rd z3tJsQcb_n<;n78_P9!H-K8kXzVm7X+4J+-#5e0&4F#meh{8FrmbZdJxyv^K+;K{&=2g5!z8^&vt;H5R6`?vI*x;Hd$ zn;MQ%l!I*<{qHx67J)`Re}$ZQC%T8ZwMBl|fV%11JvytS)cAIC*Z+OTks5YMp#jkx zx&7&zDXd>tN_A}g`C#_V=f9QKnB#dhvLlEj30IPnebrUztVDKTT^sS!?6}6_d5mYo z0upF3xIf3)uYl-FXiqw!h3B`$UfXwddy$)rF-0nc%!TP$9{(6gZT4&Bs1RaGglfu5 zgg_{TsB-r2-*Z~@VU7@f$@?5^L<=_W@oVny|L1BfY$mGJoHk7q zai69&j)WRHFx=`Diy?dY0`uxD{cI%@!QNEP??xKC> zwxYvgkGK0Dw+-beYJP>?)LDNWGMjW35Ur0v(4c{LtyI#>hrMo9F(A zy1SCsP}bKMaKt=qp}`tq)@BpZuW#6(HV!OM%_7LCT}H0wbCbsM0U zz;MaEjb~+!BKn`w61uBwL90jp&*VCNB8DG7T6gc2QSIYZ z)|fwY`}&jWL|motG;Pi`gnPa>?dcnm4R2IgLqjqTKX1;szq}U9rHGgq#$T*^@oNY$ zD;R~*4bY(LOvl(MNPF2FCO#LN8>_4FH;d3Nx7wrg&5*WaS*3(6t=YiVMMDdID@#R` zwMteu26f2~nSb>vdLK`zY0uMN@k3b`$r-q^>M8`1&vl=oyk>WaR~fxDP*iz8v$!y% z(h}cF2S-wMxNdaM&d#n{ReXiOFew1-si44SP4ex*hAgnKY+JG@ac$Hhk3Tf_TP7ZY}89(57&rW^Rh!8s{>30vi zLk!v2*l?h#DAOiAV;Bw>UZk}D@5??VN9`Eh)}l~u5W(*9aPQv!@TRJHn;#b`wNW6= z$*bv0I!lxZK5{h1uQ@jj>3|qs^M7-?jOfOkI9tZ(UmRP~&}wii9=h7#lI9Nb7j#hm z=IA=0Rpy_9bE*CP?QL5}$Ia}I(6<{*^9&O0j!-imVDM3d`8$qK z3f|9b@}0veu4iqkRuKqqxJ7|xRTCK1st7H3vE#2T(7i+KF6v6s^>J=6nXxV-#luKm zpvJ;3qm>AZGs!Yna85|rfM~K@XN1^n<)r0=Jk)Cc!~C01QftKRn7b>%D2?BAsceie z;&-s53lbx1c>Y?Kt6TTYh3jO%Zux5L2M_A2hVbhM`oqff_0?`Zxw^D^slQmIPcVAr z3YyJi?--U4mU*YsqB-tkyot)K6UhKIH2GLD4THIlGMX%w%4})Mt~44D5h@07GN8p1 zf6B|VvQ31=dHx$%Ue-5^oP)2Mjw+>zJCk(uc8*o-#=lJ)%vhscbu+!ohm$c8@SsM! z3hBr@|1;cy*3co;(@PLjNNlE0;9Bu^{s^v6Cn$6si|*S;?pb4MLFMj%Kd=_fq)#%@ z&9NGcUh@ zpjs__g(zY?*Wh$%LB$R>mF2-Tj;k*L7W?%TYpVLMBFmc9$U~`)`srS2a5m0I&13IO z^K)}aC@6w{%P7ech)2r&UYCUy_B*Trz8oM2uu~3J1;~&n8po=iR7!91U~4PD-Ua@; z9*g;-%9EywyJcj7DdIsJ8DarROuT=9H6?OR<%B-|AaoN*gE6unZo{Q7TVK z<&BhtHtcSjJ9i>oO9tMZ0Xowc#N0MLYM7Rs6>6CT&iv5m++&y3ca&I}wuSU?^LYJr%P;)2ETW~x@?NTH^S9JTtsB$&!2vu|5jNw z{utoH0G|XfNdQWT?2BW*%iTU!&+P4UV9FetafMwA81HFfaXEfj$5iN_{s zImPDoghmn6k$b7I1?zkr&*WQe{*bA3+whv!pUs{$Ow%a~cG7SV?s@$%1Xl~EjcX1j zWfU}Bj#X`tYiQUi{3@LpP0Q&)0JOYzkKNH2-2b6eZsc%j^j-kG?1pLW1z~KBNn`=- z$&Qix2hp-QQgjuTI)SGLG!DWoIe*S@k*Ww%`nO~tUvedVSC=HTz+9lXt6Ploc-~7% ze#qz{H*zr{g5=7zU#D5atH2?Ux%RvFM@5t%LD-n-6I?}I>$kvr&P*MU>~H<7p~@OP zeAv1=S{BOMRyxCZ*7q9z+z2g2mVJ2D(JvVNarfUmlx9u~-Ni)Tp!c>D+ZKBay6ApY zxk3}>HTtf{V1!`AD_LJik|?L@@nk1A@h&)9qE0?<^o&L?F5+XIS=bZlCaaF0qFyBzEtqjEkKZ|iNH`4U|xBu zq-L=0m=3`kDygbsxQ~xI=uDxB+&7Bp;a2ST*iw*~AYM=^btR=PAjGUzxwn1}hFm(v^ zc7$j-qorT;EGDKy`aaigAXFQDG))6{+OWev>Py_UT+V;d*8vmZ&QcjRWgRQENHpz& zFZ;|T3(}W>e1d}kqCqT8uukN8=~u-dR&bvp*OE3dS*ubn7%)qpM$jmv=XAZ#W1haQ zz~uNJOz_0+*Fa<%QgHR@l+8{Xrjg3tVliR z04XIRpShpsiU6X?<-qE8}N$REtJ7UJv6z`3;0%N+5M6 z9SwbbMW=S1x)I_9HAXh!**Vea*2R;$My20UoIsLTl!OPOoe#sq!+U%5aLR-V>$k?l zcHfR71@h(~X8F^a1rI*hUiUGV$!5&7zj_A=F4k~kG7Rtn^1Nd!x>h7|id17+ZtC4f zBN9ReeoDXKAcML(8rTbo6SB#09h?FB2SLb>P|n`_?y2Vg?K6T0aSqG_9MDYJ7&qaN z>BXWz9A~J5G{?(cA@)T`P!Pz15Cum>cDK2UA!SS`rxm7Ojrt! zH>0Zr20q{Yw{|Q=u8hHrmQi%0P}g-9z$S7LdqV$CuAcNK=(p(svu4zJqV%Hh*(jw2 z@igPWm@Hv=I_>kNhaJ8KFEMF7;z{sPhs{`W@Ug0eF=hl226WLfARvD|&772?yo`b) zthJ+qxSRPAotTYb76oUMs*a>Mt!ZLzz(61Wk->Nz(!Q3xsKyLEL^_dvrKF^snVBI` zif8$caPD&w8Y{uZ+75QJMzjLc#A3*Mm!-qcr%?FOBe=kUZ6N*w_`~p5%xvN#?qCrX z;|_sm>O-%m-@Nlscl0i&jkjMHw+;Ph5{B+~+L>!!4Zhkc6DhfJxe{TB?#)v^++`^^cOLmAXkHvPs5!t!Y977R^-taoJeG= z$p=zGTH`Y3Y68yQbTLMz$va*isylmJL?cVjvF0=!0t8RHBp%&0)z#JCzLBr~8W@na zu=oSW8CKTZ!x157Du@%~>^+u)9Y&wxXj%Ccp5Dekoxhe=*Nt(cV~h+>&FrPO$n(Qb zT;k-B4HErnKc`Nc}pd{wbAg^=Ys{^Q7$7lkzWs?%2j*v8j;(nze#3OO&UYe&Q+<#eCvQya9 z(cKqBdz-_@+`Fx@O-Hfc#;ZYhfYog_+l*~(LEzAsdGEk8GOr`S`4wLqq*pNr2ct1m z@!wZ%lHbb6tw(KsK)}GLr=`?LwwZqy)%Hd0WdGo{S4sXLVqgYPP2~FJ_0s)5Q@|Vu zGah+?D1cH@6uHtRW~|%m)8F(3^MiaS5;1(WZ%t47mpz7EOO`{1-N;(!p>T7nzcNoi zZ!3KS5>!KmWcw=K?USh#3AOpzm;`l`yZWz>W+BFM%rTMMX&*37Ftg#iuvYv4KfTF6 z_qA{t_Yi&L**q$Mx5C1YMMlcHa7F2{;i~!l{T1l?FWtIFEIN~Y(Rh#WV8YEq zrRUgMgK3xN_1;9eVoH9zN)$vcNwZKG{u%ziL-U(9lw$&h&)a^0*8x$y>WV9%X(153 zK1FX3|Do`vP_z^(@06$Jz4zO_k$Ht95~I1)9s00vi72qJ_Swp^OyBe^{qWuLJL07w zZ2qugX=o{JuiaF(Y7VE2V(-U8?B~IBd6LnbnXh#; z%lE3{t<$p$8NH8Rp(lB*udfFq z5P_03Des-4)8K2D88&Lb<3_f8f_YO!|BwXSl~P7W6+qwsNghki5rgG50S?@5c$ZQd zOEzw{)wQ+33q?gmfQG7u54gN0=4WS0UMa|Qnf;dqO;bIGbcf373cFtM`rb3NVRm8D zOhC&)(1zLpcqK~X?XHi9MZtF~=+Pxmu!g4Q4`8~{Mrn(ia?{}B)G%Z=cXx|A2aG+` z6py?sR(s@55Z^cFl{7y!Pge$!ozhE^g%plf!maoCf3&x^*VSDB1VJ)5oqutTniT>TpjKgV zvArBsUSqq$GNqhE3*$gqJ?gHS$UlbNa`+6Kj2#iWclb&C3N^S|_;Lo)60q0xwKWr0 zDFY}E_*)YS%aa3q%l@J~gEXmi_-PvP(8OB3Yy4@sy8b8ZAh2VkYf6W z6HHX#I6(OZUBjej|3l}^egRdy=Z?lJP3*22>voHwGIlWNKXlusZ~G-)?% z7YQyd%+HUo8h}&Mi&`sCwv>FNwM7_>)8;)Fb=}Y{9iT-Uy7hV0nH@jn##QKWzaXoz z2TsRYw>O)TF0HutaGw9L2lMB-uE|;3(*Wih8p0H6sgS=X@6golp=|wi2_H|uKawnX z;eXnfB@P6jZIlK?Z9~5Xwh=%xd2oPx=cp~7LfHn|CaK=w5&X51hfjvAr8u_+II<1j zRDD29>3m|&zeBUPhwt{wT%{KBZPB0}bdEn)8DQHFQQYjd6J1F0j3<|VdhzbT8MEnV zp@SA-VA?dv0wCECMo)4gf7plSF3mPy(xMuGROu&DN8~UXZWP;no6bq4MMr37LYJyS zjdLj`A)#(%m~Ce$2m(|pESoTe5+#wmvMsJo#t-xw&c>UpjnimWfn}Mo}phb>d}8GA9_?kn7w;*WY<~=yY&4Cvpt( zyF@R0sC=Ni`r%*fSzGbzUCRWX)?kR`E#}(PZ`W>nAZ-f=-t6>1+)L}vm_nqHM5*J0 zA@t*g$n@&%&=2B@iHf4n?%adwW8_RL?00k4?+?55v`pwG8MGCD?ryzqaUJadhko|A z4)o*--#(hK`7xG=MzXgp!hXPB$YmAw9wC(*S0<M z6YCIOn|eO;&rKZE z$fYW;W}+63&}sM;A{`h&&-{6ZbLjj(ga4BJhB08Y^HT5&2~kS~5CpJvp94*3j;s@- zXJFL!qqDJdQ;2%5Ae2@Qd)-?E_hAG_En2hQ_*BcUK4q z?tk(p5Q|Y)jpu98QyEC{*)~*N$9#jn`robC_usWQ{=K~&k9YCp;ViLQj}tRD=kGS? zbzka`=QMHyQI0pucjxPxg#48H3>;onEDAtOp?02&G84%mKnIas%XT}hW*~VMO zfXNDH2+)oZ=GSn|r}uhnJ+}mT{nl9{H8^Nf`?T7UKR=87muEX#^gg3_hSqFsiAET} z?=t{(`Kfx;0SG;wx_(sy!xCjYMsDtDmS*WoH0R?Ah1fL9AKsb28S~U|PNa!ta1*uI z4hDQl<>7Z=WxI5u!JMB7N92zq!+! zP#r~W1s8vm;`lz0z0=XqfSvTIxY*6bWure-ia=XKp+N7w3dN<3&2yH?@N@=tS{vmY zac8&svN`u?dTU*r&98l{Mn)CPwCco238CclkpdN2@lD2$1kO`a`3?amYU=02IsqN66{!5ybhA&I`pxds2Kt)x-k6}CNGR>0gqWMMNS7Ec_tdNaHjdgnB zzuYZv08-gH2DC6wqzk_*=%KEp^$6M-uGVz^(-$o4x0@scc+#%C&RW%!<0%8fMz^Vl z=~rk%*;>n(B6p%Oevt)q2D_=+8>Tx*Ji-D3S*_$Z5vd7c|MmztuH50Sk+(fN(2`TL zKDE%Zs!4a|!OgSw?G@#mMcTf17ze~9z<-)O_S3}-p`_K-g@y_z49}t4;GuI?_={PQ zMmR?j{SO$6Zqq_$tL1CNCC{r!_^KAM^=Z;bB8olS9gwAtS^S7$#%xH~UE1OV-_t+k zVZhcuXU5mqwt$UtY@3l29j(Ewux^D^qLciVEn9@xYe3X)g{0;MseMU{NP!ChDoO66 z-1@ON+dJ9=#m=VFwp#9q^p%qG5-_ExSFH0lD@Ys~U6H+iTuA9(&vGrRjRWiHhoFk{ zDOK7zf<}X7t{i!j091_sAX)Dto~LJL&3gj`G_B`NhxvJVc|}B6+=)XIZFSNzu}aJg zb?a*PwlQhAvH31izvYIJ0IdX=!z3L6yHpeCR%N6DQW#NYgI7zA zX%&qF$wD7~yWbxDBsTXeZEn8WO_5@B=6mGN^!sD{qdRotxgPO8{jVoR>XYG=v`d;J zIO6X*P+Zk1{5kY6^TE2Uo%pQAc)onO3GuJCeqPl6X~Um#RFDk$G}eq3>O_Iu9fybDd2aV#oMtoK9&Q0oFbkf6CC5~1s)WbVco{NY zPhE}QL7u()fUS|%$Zqiz0~$auyd7$4N|XgH99v@XUGpHWv=Xdo)_=8YK#RQ>?2ny) z@je&CdY!BR)f!3i!O@YX`j>o$B|XzBZG!xa=e$$7xgFFgmEFz1&$_FTjZ?I9g&A$4 zKe0NTTiksM?tHJIW3f8Xn$5Z^*Lv`XBY?TSv;+H=u2i*jHcbEx&{z#sZPlt`PDNxj z2-6!Ny_REH=ik$PrPJV;G6Yogf7;H_yQe3Nm?=9@@Q~!Brm4DuMj})o{}D#AtHr)= zS>f=9)$(0UHIZbr5Pm{+nTTim#~`fHinyV$O_y)Yui_o{PRFLdp-9ul1<>}NuBW(w zpEx|bqBieK%%XM6)H`EF%vSLx=iq{CS)Y@&;9uTSw$BVEc-V(GfV zOvi_n7MHcJ0?y0s4YWqshlkZG4QknU&hd02?bWk*ir1NhbXLd_%bch&Ri2AjuU&!n zu1A%+y1Gr7C#R>%@@4Adn4KY!_mH8<-Z2p_BkF5OhNeY~Pi-RQ7>Fx&KEZ%VlLmyN zp{`VABMni~n+hfrC(KN?Q9P%V1U(yvREO}TYmpclC_?uicf^%iDlkQ-t^utDz-J_6 ztRjLl>54W}`7EEi4*G6;eu~vAhKC<8Qy`;cPKfnBwPv#}xj<>ub+T=B$C!_nxI!PN z0ELn|=n@{kQ0L?Bfg*1L3E4i#;P2*fuWgVme)nE)ehp?$++q z`IVm=G&GE)!Z7uoapWLxF`d-S?zp+9f{5>WbkIrZ{8$yk$woqXctAM^F!3GK0rS+0 zklm(=(dDUiwT24OcMyCyA0ZQ-1Xph1YIE3V-b;pM^;Q1-9EpWqQke}r%zSSC?Q3o# za=DvDz%hf?-uO63Hy~b{RW;lj4ULnlSO8uQursCH|2cstpV*T?)^c~3ojKWlTv>25 zh<3Kk3UVYmjk~T@%f_(uo!OJdmmJ!ZqaPDXnyqJ+cy98@qGVPm^XlA~*sONH;#dJl+5)mtOBvqH(>lO!f#%+-eL`ED~!Q z8ykSG`TE}fFLQIg9``=tHcEB50d44_F6u6IJ_eie;d$)q%j-0{^jzf z=*sEus)6PoxAavp(<31X39Mx_RpG@^JjkAYI^$ZB5v(*ynrOR@ccGmGsr!1NJM)3a zHQL;P_r2N{Pg5?M@IV`cN4n07<+#(<=~1;un##KqhA;N&J7;m-t-5A zxCCOZzUxW0#=a*fd;4l_z}QuX8Mc2hB;>Nqb5zUzlBgot_dm^-xe~AiwCX=+dL{|$ zIZSMvQ`&KD6gbQtdE%&%{KjxX#YQXURbg4kI8@HYjLLYsU%)$VAOUru2_mzhPD3MZ zh1rJdUWXF9gk4KCT*SgJGGx>l-QLkko&NE5ou5V z{CoWq`g&^dJ=-vcC}89$vJ{g*E$bz0OCp*A~Vzl4~)J zCw(faikYfhUGA8i=jHFT4;ju_=hBJ!;y5zkovL7%L(IL5<&hipB-Bu2D3t2-d9}3K zcwy(9ycvmMV$nt)M`a9%7WLovUIYW-CK9P!&4>MbuQb*O{GYcKs}*R8LKWy#LBPB_ zx-%SK7F}Fg+IG=Gpn1=e`R8%g7wCa-8tS$b76-bYwQIXZt$|&KCT**uvY|GF&evRl zh)o$x8rCv~yr7*~0=gL_+OA$%7RX|)%Rbi!=usAl?!_*ThLn`l4~cK>QUe0W8|ZmW z6bp8x!i`2$q(_pj-mrV4Qtkxra2>$NT3=kulw#5(Oqfmt0%Z|RaG{rx)c=e8kta$w z9|o2D4p*z9PiCOR)*jFHTacOQ$1s{V=ZxM$U?!_CwC;9RJn1CRM*t}egI=I zwXnd+$aTxMy%5!F=~ma^mB$Xo4)z0f3uR?x6Gi9HdIVgCac&FDpq_3%69QhJ_lWm< z?tC4*jvmEpMAq9!Od=q*Fx-t&WuoXR8-$)AEmZu@RE)o?P)o&ca#6M4>Qp!JtsT$b z(I|?_)1Z)(lhf7WI~l-+e@cUQOx$|C52i_%RT4!N8YXBFGmOa?)lbpcJqFSB#LCWr z{zk%On6;i2cF|nHaBdq95f=ltN2l<`LBgireCjJS?A;cbJN0&Ya(l6}X{Hyqx2>ui zGtNE`&A@uf&5hBuBRn7mcs2fDhoK=l?a#nAA?yUCs!Dk#C!mj zZ@bM+FPO1oy|UxlZ-RPFNNR9?z#0wWRwNE16B8+`B`=~To4 zE*>}HX0ywh;CM3-?*RA#4-)1sGX5#R9)ZxdA7MvwMZYl@GQ_~naCcx}08m%EySwi$ z{%|#j>jbI)ifu}&`3@|DT9HE8d~|679IGnSK01y*Q|Y2RrrBH&$SZ{?rM*t|FrYx4 zgbN*;7JrI+PK-#$8u*t2NeuvI3B7&$J>ZV{ri*sU z2WeOR6kFZccm<*@)Z%_%R3LhSN~yYqffAc{2j^Imf6@2M=|y^;sq`(u0Y)&QJTBXV z)ek7R!>?id2H`sJ4J=aWJZ0Qx$H(w=a7>;HC3{ltKMasVu2eUx97!GWVo5%&tQaNc zKU;!fww2bBj;+IN`|GSukN60sYrdQa8~vO1vY)K6#Fv1zjxccl1TRAz4ZXWN&}Ve3 zHA*}366MaM^Fp>hpj)D0oKU}QQ=5l8?gLA+18y=LDm(CPDw7kGVm?48=)GHY4&pc@ z@h7jO-R-%WZEkLUdkYyLgk&ly`JOi$Gfeb+3*wVRw*sYwyuJ7vZIDy{5vMaJ`m_f# zdePabgS7VKnyw@1_K1Tcf>jD75A$s&&7YwaRC0BD`U@dn=`rv>n(UvD<32hfz+o#J z-_ui)KoG{Ib7j5g_0@C|*uw_b*Vh66fk$weoti`1NBG7EK=p46Qm+fe{~n7~KYU^_ zpQHb}oG;+)&Lclg;ced{CK~+s zLjsuJA|Cx1`+kaiKV(X?mGh)#d`58lZDd?p(`xJr<5sIN>BeI%F2r6Tdy{CQD{AkX zjNoIs2d}O0v$8kcj4aJBUBpSWx%V0YSKE?&2)=?{WHC{N%cMN*<||o0U{I&ZCpv+V z|1Ny^&LP9-^gOT1)%&oMAXnnCj6Z212)>N2M;;!M>-`loz0 zTE#90GmoWi0uhs#ppw}-#o?&p38sZFQ3I(>0?Cczw3e!vI*G5G9Gf_gE;Q~A(Zwg2 zpazzj6lH|~Q6B9Dqi!tRPlNLc(BS_h&zF)m+<3@pOH0GWy7hdgCs|)arN<>wII711 zl%0|ppZbBT=Mtf9F1=AbGvfn5svCQ*Re%N9yyjD@gS>CP7ZJEyzJLI!b|tD@WgVtJ zwC`iAgucx%qMDKP-PSB(+JAz5mW&{J|Fy8F5=_DSFu@T5_RKdtII8dL1jpNoYP^{$;LYiy)k(K+OCDzb% zxd(}Vfh&j#$pf)hE!zfn5=umoU~{Boi39*c{Ix-Xj1Nmw5h~+hT-L&K0}&Z=K$2qU z6}X>J<rSdl41Zr5zHcWt&WSes}$8s?QAHqFb7Nzg6C&|u%mdkEV-u71 z<64U$w`~nQSz3b~KF=lA{L{gpN+xBb5;4_0q&=M--OS0}%zwBE)h3^~#)uH)*^ugb zSnOW}X*$b4l-RV?F^wvocbl+?aI{Z;DPF%AT63yj!@$_4p!XLhB}dQ&`+$z-Rl%zI zAd(e-1=P>%wqoW@ID@=E*oEZDl($DCAd(Bo4iUDFKGdz4uQv9nVl+eYv)Ni@ukAWX z!c4*cadZxTb-!&K&*sV2$u?H4PFS{=v1PYq8_UKzxus>>T<#Z_wT$I@zI*vmw)y8gA^UsC__dK+eBis$+fN0Ts6{ zP3~s4Q+^KvYBxWNUGb?cy4oF6p7+#cHnvCGnw_LFbAjni$cVqKW`>1o8Yc4e(?hik0EtIyZW7KlCa!v zP3mpsv5X*i6t2C;3TT?p%R7cDlYKrM7vmD|@r+l72{I->K3ibin}v`H?jS-; zl5rIMOXUdm1T--iIU_Eb1U}Ps?)-Q&I7|}Mybg*QjC5eV4%S~8X@oR+RoRpDlo2IA zWXf0Gl&i5Ff+G``t|(*kCop9tMns(bWHm&04C%7<4<1r)#>xQip6mh9j+Lj!dyO@p z&~4P`jlQ$M z0FQW3i5h9Wj-Y*~LsnMcJ6n+a zEjKT(>t8bX(pZic9&D?uQ=qfWZux|}nPGGOFGsjPB3Fl?2GZ+i$NOi&ehJLuE}odm zBGiIO3kEY^|4`pZ(j2SB*GepfWToFdLoQbocdQ8=tEAoivK+Av7RTTrxxdE8wb*Lu z1TV2=zkVw?W~wqv&*#^an%D?TXrw%%5pYfuA)rT%A&(E;EcT zF3cWxP|O7I#^Rz+W}vvTmj>cYdbB83n=;T;J~sxBjeM*gD7c`eT$z|9r}48=eC9ET&)ps+PBfuyiq>R*rZmVk_9zaY$N|Om z%!^1){&Lp`)U;CkVIZ?t(%eeIHfg#SZVN;NBJ_L%1{EkKN7<%S$>FvGy3O_&psoNC zYEy0g)jRo)>O80EG*QY9k3VUn?%>m-EmGeO1cgmNyQp-4Pw#SO{IHmN)uJfKLibu7TMDGN!QAk(|B@YQ=bvP3Qc_pgz(~`5eX15QNF9$jo6Lx}l z3TzFMaQZQDc$Pw#b)?ht?hnI*IDpv(QxfbamsNh@E*1Q=b zyZBj&Skc%(nH+KKDJ9J9uI5fmUJI?41?!Jptsy=4CPwwWjgY+yXN@Tf&WI3rR8O29lrWK?hw`|*oAPtr=@_G zrMuq&RDR=CHDzDH`ox%*sIu_3d6}6HgPi`C0N#b%qBpEtexFvLTq7|3z2^nY&Hsl? zVOIoBe5_73B&fW)ZaN_<0jRbTd8EKTogvv=9g1yuv+p(bSbZ7wq1mzIKZc~P+u&(l z85+E^D@zm~9eRXNPk8;7zg2&t8PtW(z63oD%8iE^5k#l+z=bBEX(&;%T2`(U?Qt5Q z6zTwpw}6Me^dsrSBV@5x)>f6ry zR|{)v6vo!@IA$|++Yk>X#9!i1C%WB_+J^<+Z{=1qGWyg=W}I9j-x{sjen?lT@vASV zMtH`?qepP)H@AY1|EB5+jLA1x+uK)mb{d)722O)aE*Fpcj0>G)lkkruJ#Y`3xNcq{ z)H8K4pT$0{a3C(2TN6bunM6yuMm3uS2SPq<{V5|by#REC*!Xy<1B90ub@{ohAIz1K z&x5(Ziy^_4Q6(qv9?l&w=MXBAqFKSK7Z(@Bis|`{{ujN-Epe@%fg}r6>K>^CgX#06 zlfN4>3+4Fzh->@fGSOE;jQPj;xo^}?Vpg=18$gW-tev{E=GKr zm*;=qtOk=Y2My9bpsww<+dj^voyOT(n##2I1m4))8-)BsiirN{3aV7c2ZAxwGXng zT?vGl1D{2bg2$3Mg%z!&gq|H&n@&0BAjNVc<~D@4;o4y1*af z#@}=Xrr2ppnl;l`$x2CnD>#!!drNV(BJw3;G6Jl#^}Y05nEgn~|JuKDHcZ#}o!b3F zw5-2gxJTMIjC7wlA81LsvBcrx*b)=XtTY!Tqt6w4(j=yvuXeO~$efG`4{do41dT8D3LkqvR?O%r8-TFZYFkotWM{Ja{TA zm7hh4+yOvFyZ~JvYMAcV+nSw>bHl%f`W1R}AW;%p;yb4S18%dtvyH;10Y!_wv=)YY z_fba0AHf=Bi4f*s0Ys)TT2d#)wWT&{_s<_7ok!Z!c-l&&#J6fe&NSoPl8CXXq28!Z zKMM{5pN5^f|}wx$l^!J7h2`-q2DlA!gqwOEe5aDT#GIjU+}#`%yB7fnD_ zvaz9os|Zz!&I>2K{Z>XsR7yv$;Ic}s{&Ctz+;xG6e{lA?OJD1U%=Yw0g|fm28Rw+Q zrNRQ75a3k>_$&2UDMEv*T01baLbUVV(1k&s(CSg+^|OykxD(Tdr3Y!ly48jcHE)lc zdf|XcfU`*^VDEXc32V^XX-=rc6L8r(g1(M5p!a;% z9Q!m2hsYR&X{?R02v0_(ZI~zdq$*JH`$O%fgO=qYBLnKr+Tcs<8U_x4da~c>(B~w* zqUG99Q3PfE{02RzVIMQDzgl@_olDOjVZf_<;rC+%gyWj-R}p-56p59E1pipattJxV zT+QLnWK=c1kvDgDQuY5-j;`A(CrM7kpXmB5`Fm%v1qI9lijBCB7?M*yqj6(A3Q-q2 zHi^pUlH8nh6Pl>KWK1W(|KVn1l4?0v^YZxnVr|Uo$f&fH;QUZ1y7HevMB?X1beE`QO2@mQYd*lq@Eb5~fm{d-OWoh!mfw67 z6qn1tU9U{Q%5-B{Qsj^K7(o(YlKrMoRSbdq*zbB$(B)dN>oTrC0^7}mFItlYU4-#% z);%!+SvP3Jfj`qe z3x}!|1WFx>ZVRBm4oTx?luA{7_|MM$P>~cxrLh&xt<||q*|#iz`aLuZVqy6MT_2sn zhOTu4M;cYJy|a_zi_wPFFjyqJ}l_9n0l3V1VfNdTvQCgq#*8|eJua`o_BgfEMHG1cqQI2T`^*ry2 z)g^>g2nchM7Hz6%g{Is_?o-Uj8BIrvThuuxfLGCEgIm+rxic)s-JoGzcN;$MA7|qK z6|1;#1X5r?qodkk)2IJhz!N9R<3LBZ6}M&+dHB%Hx*oXMT0AdlsWB~*D!3bdg$Mck zBmzzzS&uuQC5?f)DIHln%4zjIkkeU8T+v7nwv*X^)+hUEW<)Hj&DFfOZBj$h$|I74 z62~<``p`yV$VW|-wvzilc6Z8>s8X5x1*Qcw^L|R`etxz}94g-=AL9N2C5QKyS zp@NTt>OWjEx`he9=pDj0pWOr|B+^&o>Vra~6p(`jCEmfc55oK@E-=Q9Mp8v_ec&AJ z`0;l#OH1kt{D12xHN|Xz=n52>%hY~7?lb!tjeEj`#_BDu`RArZ8 z?NdTb#yv#C!N_PGxJ>^Ee1N2WKK%Zbn<+c_D#;8s4Sd#HPZjWd*?!)&IzJz(x|wdq z7q)VeYE-9IzIa@g$3EYjDmvn2cs1p^u-P(TPS^)nB>D$p)Rva=1DSX%E4d+p&UPBs z(sc!Wt5cqOu|(e&F*ou+pJ)R z`QNAo@V}0K1zCC%3+9J9QA_``a!;KCo;EstaGG*<6JBITlP*Ez*qE4|FMn5omaCyU zjiY>L-IvkFP*f*+*L9xeO8AG58pRWC3!(JV`e`w`6a?wfU(E%WP?c*<7E%<%Xt7)# zin;#Py#%=NaI9Sj=67LAeP)^VBq z(~$0mNhi2y7!V#wbiaO5T~s;&)1LTk3(0}m33LI78m4m?nF|RGRvP~f4apXCqI~b) zQCf<8rM8NtXn0l{M!lO5#j(>Ue&ZrP$iso5EK!|S5hLqcdg5^INK082BJMISz{Ct< zza_GST?qyQ6`lP@t@V|G^!H=va!cT!(UT@UtG_Z&?s&D;>y8zj`D~l2JlCD(z=G- z{(hF0p*@A=4CQEH`_x_ev^)suqq(kdA*Xw*7g|8z4^ zbffVeYerv7$rcCVWS-`!E=doTd)Ls}X|OKB$TGrxM(*!jX~eB5xiRYF@?1U zn;#ZG3`OY__fY#26Y31_i5Z1>mL-Xy3lgNVvC(61EVaO4?8TVxxOORKPq)foVR5m^ z`_?wu%P}?!nbA=4r3_@Le(K$Sj7JojoSYo1nXZUnCjVJFFWvz8QZdbD=P{zv3{z6Q z!JCkLd`h#;Q(|P`1d+gwF7TThqSJZ8c;fjMDL@=xO0T-!an z6RKM|jE`fZDWPkX*jL0l%gz`H-4^|IqpPH^#M3xhA?`XO5T!APDf8%ggvk&5EmDm_ z7DWU}1#qBPJ02O?Bxv?!-u;Kv{OTnGipJRg!07A<0pw(4g_;1;A7HABD5*@h`*77hg?a0)` zmUYdpnqN!t6zUMTt=4(>lo5QEw=MejvZJu_wJ%u2YlHvM&h0unUbr*5WO!!6#M?2$ zKNqW^bJX?aJ6iB`JGKRoFMHu7MOt{_7vHrr3s&N#lodlcdkiRkWncr*LMqXo&PwEGA(`lesK54L&yAzpHHaG!!Cz=Q&d0+%`7 z@5Mcr$9&E#5fo1>%e_P)M`1$@eZ=!@0)r@+ew=>|138x~AjHl@)@qg}BUKG5uKD)u8*t^46A~GWjqqz$W~-A2(E)|#7!iZ^B+Fzz zdBX3Q_kK4sA0}QI=xAB9lg(x3wNpUf_nE;&<+9tnAO9l@+xALHnnbNN@YWV$C*na- zMhRt5L~o~Etxv)~nIhRI;Z5`@!;8hkLGW2KV8)XS9uM7C4U=X~N9makPDnI$Om;0V zHmnc%1$=l8|NVr$HGA*267C+=os>!BcbokEd3UYVt&*map>v9#`3t9;2O_770-8A$ ziZ38D3c-$0Z*Z2#K=Kc%g3sW)e#!~j=OtN^fs(T~iK0<4Ai5oTA=Yqx-R9M1qs<9v z>2D6Sci{Mzz0}J#S!M7lX$I|ry1W;y3MuAbPGPDguEw-jPqtZ}03V;{(~{WH%b&eS z*SgKhfEim#Jrh2;{4had7Z#v?1di66)y|K^OR^5Z57A+N>sG1}lm43}BxS~yw5RdlLxFf>MUrd$5Lg~Z zt4$ZS*1;bmuuN1kJnopm-5ZlhX+{p4X*&nU6{sgL0^?wCLJ%8-qJ@4B^m&A{h$PMb zw!$P=%!g3h+`BQ%K3jTbK#lt+yD4e_&Ef?Ll14L$!aAglegZBZ4*hWTrV<@I; za@K$@84rO5A^t9xTVAeMCR&b{9o55ucC1Kg&SFzim1|U|+Aym2b6ZNnUu*K#1AZj7 z8w2=qlNj}bABfh2YC(Xk0^4WsQ4(4Qgr4X=d<#Si;wZ6)qa0!DEgcbOzHx?k7UJc- zCJMMD`bQP%2724cVe~8^Tn5NT5$O_3o{a3kOq4L`c*`}TIgE+z0y4*;4jnXSM${fQ zqgj0)Uk4|94tUJ~prpVeRT(QV%(C#VDe188uyl>^h@)EBsb_KQ>``3y5ON%5jQesb>Od z1wY_Vjz|Du-cwivE8_iVRH%DD(hqR;g1t@wJjq7`0mn6zsWDN5hCAw)XmoQp}O!+{n{jRab50enL*%&UO?h{+26c48;Tp;YHY15_BuxoNl zOHb==8P66$rk1EjXZj5YITZa8K;0#3 zJmEVpMlp1wRohStk}QiyAGAZEcPBIF+9l=3#WB~)|7MFr7+n~Bt*hG>)y8B3CywbV zxp&jWjjI}qD#hnITSs}swhyZ%E(aC!`5Rf%m2#C3ih}abVLyymG70bl?pcA;O|O9bv|T?Z78(SHi#iFT|THyNh_C}9)~@weD(JDI3a|*Q)Nx2o?gzgHqb|7q^UbIHI+ZL+3RGEwW`SR7VYsRk2V_v{ zUM4>PTJw!k3@tU(PkWFdLWFA+#9eqzgVY+9r(_y#X*#03+10U2a_++>pRL(FhipKx zXG^2v0eZLD1l$5p-9Zrw6f>}(x}BW_oiGgq8tJ?{~iSkMU$U)Z$Hbhzvk zUy-mvs!EYC&}t9g3-+cNh>l499055gMyILz>=9^yi!T3V;NJF3Ug__!cV|CuNX#h5 zsJAD2>|;g0`@teFus*@@B4to^Js!e^?*fDHx?u7`AdVKcCe-13dFpzL%iMsNuZ4|` zEhX(d+lPrd`}f<|{PppcgQA=3C$o^T(pu|;ycpsdTzt-cvxJ+LE)j1`+@Hz zLB=i4uzmw!2d>xVI_xRhBb-pe*Y)2oMnL}!0Q6gEMUe(2Qz8JRB{$eXw1W1u_(vKsXp6q&o#A}89=%Xfd%R!)SqumsG7T~B34I%$2iI3%*4Ly&wki7zD=G(>uPZDJ$&74S zjLOm*xtP;QUQ)v-FX^MOf{{Q5#E@5jX#HYKplQf?|$9` zfPa8TNpdP*P16>~JoT-Ais&tFVpDO^jhm=9@6ull{wFd5fY2^p| zNe>_)VR!Rksn-Ig71U9lFCfAWVJKBK9jUVWY-ca27Dn~aStgAsb&B{E0D{K1$SRM% zpmKjosWKq)C}m=Sr(wk;2OuD(7^W01?I=TG(`Z+=bPV|yp^paDJa+l=M$$QKvZ|6- zIqw6ygxMi`Rx)&2Y7HY+KllSN7nw!KUts7TfVdo>yTHe9dK(r*+GdB5tP`--kp~@$ zb){uU^Gc!t-5hVgYBECrP~+oih1yOfayF>naNVJ`T(E$I6^`|xojSQST#fvbd{?W7 zrZ^m)3C54+tA}hGa{j!N_~D%3Ay_5H1f5-NdWn(cJ|M|rs~i7ckd^kzYW;jZa9#}! z51X|bFc-&#i0I<73$cRMHzwN@5QXsHT?z1#tQe!_AC`Cpjz+W!Lhz$&pAGd6iV*#Z+~5^~S~qfIlQGp|&o_ z2caw!c%*#~ISH)we0h1H0yuL3Ev6w@r@~soSOk@#<<7Il;I=#s3JpsLRT`OZ8*nJ> zhi*!5Xu^g5T!AL`#S#K{rj~R;26<|uWI-A?3viVMGUfMY;kV*!l+ZG(AVHmel9xVq zu&pEjfX~01JU|E?KOB%zKHY><&X8`Yk%0^vWf9fzFvwWk?k~2WP#SB;L-L{a*`)%+ zT=n)3b*7s$OkJ=Oh+_ioMmOx?8U<2_e;ciH4=GsdCfPkHLvPO7&x5O+3$wwux{7OF zzC)Y`Ug%}U$n8E-t;(!IGDRuGjtn2W@S}B-HQ+)`3y2j*VH^m8)C$r*rD`8y6&t{T zVS?Og3F;7Lb?#b`hf2Q@<`lyOcS6#pg8&A)l%d3jV3KgvLb%O$E7{WTX zf<=3PQn;x$jUOn1bGf^_>uX^8VP?jh2Qz_{F@M^8q-9GV(7iC8fV=hR-Y;{uOnolN zelT|NRGn)8=LCi(orrOHo98SC+I;3}a`^HQ6m%v}bR6|dlC(U^qx+py3#53qfLJ=} zPVY$U56Kd(xtgo-2-=K{qpS4tq*@)|wyS}xBby2wy*f-FpPIw8DQqN~am0sn`OMNG zFCn!ESHk1AP&GPF4#%7g zLQ|-r5P~x{0n~n5))|pD6aw*$cswD|ps6`4UEJwq%=U zMA;fR&o&ttCU7QtDXxIk9s;G@N%#aicfPmVnqA({=YAsp>LppFDL>mnqE2z`ti!yr zN`oQCH!CAR^Q)A^KClKb(w_buGeCwiV_t2SE4SH{WU2r-p>--m^O**7M*UX|SPtA8# zPCh2)ogKl7F>JF3_>M}fzR&lUf*YRjEDdZcJkhS;-{p>l``M(SI*mpqfvXz20$B>e zM`(@j#?cxb6Q)gFO9+ng@ba0S)9^zWD7onnNl8=4YsF)a zGdNIoM(;*vmUTXj*?Xe$k{8y|c2rh)Q<|LOkjtp-h`#)7F*m-vydGeT8&LovHHGJrfvuSFaOsFYAs51*eI9Tp_A+ivA^lQ*yqKr^C#Ss(efn zC}&l`Ct`Ck{=h0Ioc?Lc%fJy6RgOvk+vq=~DVw^+-2z*m_61BaH_k9|hdKhO?zflo z;o;%m{!b1gZ@}aN=O4B7G&Ekf(`|`sM`(fUVD9XE0fg%V0;QUH$31Gb45@Zvo~tW8 zJ9!Ec3F{WHQmB5~ZS#h*wSvt6KLdEUvJFq5m9lB*M1Di|lp@{$mM|>WvFm21uFHLG z_{v6?Ua5LLDe>-b{{XNOfn3@9$uaZr(|R57J>`plsOa<5yYn1TucYymTZaeS7UKW3 zF0~yU*l`r(NU*6P)TmO-?Czt8km}%O9=MO*S%QJ@-s7`R<|#10TwxT=Q}d1O4Q{iP zxr)2X6aAS?h{(?dJOEw5F5|O@5WYzjBa4H>UQur_!m&|8iKPWy2|m&cLyC^Sl-q@9 zHP>d{AElc0y7>9M0G@~2t=?2T?3^cas*wXu_&|O|ZusMfLuzHqmg8zH5%*vPvL)Ls zNku!I7l+|^g1(6G$1n?LN2X#W7MchAig*6;b~Nfdlk>m)|MHd*6qd)f1M(tenJAg~IA zPVd$hc_zoV8nhZj`O?;S2eWLG*okQS_rX`dsQGW$!+!sq@DB6ONIWCG#N-e4kNWA#( zCDZs@wq?6p6ea7ZuQF-RV6G)Kb}PREP0V$jh{%mA9zXuASkf1nt}Cr9`di?~On(?P zv#aWyQ4d_suO_d1CLD$>J+DPS!o_!=5zfnL;pm#R(L=95rwB7l30{x}+-|$Y%H>wK z&r|8?xZOURJ+Kmqzp1wyDcNqB&?C-H5+=knSX^naEobKE@uidH53=ZCtNjO7&o4KY zN-SuML83 z?ez|TB9BkaUlMpbD@#aSez5>?oMC*>=s0X;(NtL3cYR4rtFZ(+U20_4EtYz&%IZHi zJAsV#=Mh?*5qyR)J~LD?hyfYjgSEQ%aCAk|R_Whys|a5=fWW^5jf(zNU3=1*v>4`* zy+J=_DOuHlkc*}qo_Y^--V__6)8S&H;{=vJxyF#cTCvGrUR_HgPW~YXNfLPl@O=`S z*uV~^o#A-9Jvr}w9wsGmT4t~K`7u8S?MD+{+&HPKbl1P%K2sxIPs}JX!-p~fC9Qc3 z$rSmv7!zv%&;@-vSY11g`)6!83kCxe4zih2|VOtvP zpyz)@vJ;a(ly;jHjuC9SiH8Z&4rUJXeKIpxJ@mhFgs%&!UJeDjAL6Wb$&krh5bG*Q z+xm?P^Oowb)AZ%Yl->IQ*X+5ijF~7DaC+$I=y1Z#!x?NMC6bEs*)7pUh@u5wepF7f z)^#>MpnjU!r62JoG0M#wAoJOxD8*4(%YL}s+S(HQ?TdGH@yF!t5eUAnk*KqieINVs z_s7s7=SS4Rx)B_7HZx8Mi-4EgB^uP;diM-D4f1~C88rnKPI^1meR%?xLy>otsNOR& znGaZ526lFK{zmgD=n&uh)>bc9*Q6Pj6i}VGL6MFmeX}7L1+S5VL0;3kU!XsmCVBaj zqU7u-zEXyA8#M;K=@KE`$?U(w-Xs8U5c6{sw~dTGcJ3H79-5NGI1?83Sr=Ez0SWkz z)Gx+9r;T%^s)Y~0V?IsE%P}BaWO;YvjQ90NPW&!g8lf$L7^_@6DjzC0@#xVmyXwpJ z`!_dzY#1jPnot_tkq4?}kWsq`?QBVK7mob6gNBafJI`_dbgu?AyVRd92}fPMWn#aT ze^CC*aU`pX2f3gue6OE1SGE0?V2PeW`1{$4?Pg4GLgh+l8z%ZO;O#Cz3cV8udcS<< z;dJU`neR+9Hz9$O`l_?|wkZ9TW}Kz0m?~(vGmW?+4vx7ID0cy>N(#X-z(D}kSq%f; z=fMGqeqSoHMB#|oa^*OgNm{pD2a}O_Qg09~t}V?a=XKgu$~z##553Dc<3xILvyll1 zcrS$87VK8aaQ#P5>b8t-nkj6Jshml&1JEb`{l=gaa!4++`kq!xpP5xD_>a~ov?LR#Q*idN5V7b5nM+%xu;N`~ zbS|xosG`U`TL4U%AAEuwxqot1qDO?ZZ{Fg)*F)JYlK3Er9*SA?0ATw|P{6$Rz8oY1 zR)yT~?P#)mI4m><+opGD>(u670~DA`r?gLh??p56G*oHQMR6Qn8SQG8mct2cXT#T`?(}33qOCi#l}v7_0$@di2q2hNY59|&^^#Z) zvQChb2=^3%_W1A1zg;qkkTspPH`V$`t8Q~C=<~$lDz6~9hpeAf{gbC7k~SQkxIZD( zYmr007=!~^*=X~yb9bM&e2SCPFv3D20DaBVV)eB0Q{SK)(NCmbuz=$shXJoCC6km4 zMsP&ek_iyFP+G&Pyp}r9Km~@FR7>_Qw)(loGD#z@=T)u9euuAAFxI74T60#9Ue$>e zU!iB4Q$=}pIQs%nsej-3C{DJkb^U_^J+(Au;5|^D0r(H3B>@WPUeGB@_PN6|g*8!%QdG#(u(E6yYn@Rq z_yLj40=@PsDU7_m2XF`Fj+?;CIpP7(3`zRzyu5Hz1T1Kr1l#*m`irIs6$ks4-$tZL z26$hi42BvZdw#5zUXFV7hdDoCek-CQP`{2XQY^Y_wf^u#_(XUR*9|QZW3WrPDfr?N zJmk~!k>%UpIYu57EA3UH5Rjr2}G!6hrLJ$S|Tnxfwe~e zE8V^oRdJKHyf&4~|uxVK`)$6tjs(5jd-z<@WotnlPut=k^z1`;s43!U~X?bn5 zTMCpasX?aUp+|V(+a@i30H2m~)SE#6&Y8cypG9tqlC0{R3R{8H<8J(J5&TuwsCAQ@ z_wFl}!6P6i&IpT%iD6JT2n4G}7jnq-WEo6x*e06mqw%%U-^T3n(0_oh7-Z@O4EcXv z^lNK^qqP;g0?V>uykV_qfhxhdo2+WA^k@$Xg!kRMcR)+daQT1*0V>}a1}Fg`>Y?5V zFK*~Ws1~WtXw4Ziy#6n)E<;=$pKNW>im>@})zJ+Pb1fg#_b`EYP<0NzY@DIKq@*lU0LB3M+|cS8?$k5GN6#($z0^#VPn#Dg~_kq~|pRngA|!RTXUP;O0_2`poYnNCCzDoau-2qw(jLGrHPy+2Y&a0b$>Yi-pN?24VU2-} z4FGJ&)qH_-;_hTwsyGa0b7RN-!JxjakqbR}8g@#xztXjnW51t`A@4bz}4TMvXWG^9l-6jvw1TQp;tZE6JdNFeBuTh+Jpt+ z!ruZxpr0iYW5i(RG2pDWUu#L^Pv^Gi&eZBZ;Nt9ZtEBZeNS-VNMT>Y)prZ4TH-nr0=41;T8H?fXC%$5MtQ7J%SSU}UqxVZFevu;YCk&xLv1!O~Vn+*Q`q%y; zu=M)Sx~9U$ZA`FJs@8dLfJwmpMl3LxAhJx5vO#*Mze6lQNZM&KPvyt5E}Pztso`&9Dt|;v~xhySYBIGIz{ZRWaNJ!oxDPavs$RTug-{u`*H%j zxbl}Uu&@Ac5U8X95Vi9w7e+CiPxMN1; zX8a4{Cu0sbp_zOq0JI93AOd+J9%zK3e35_}3gPO@p>{L!{ho5dVGy_-A4K5-!05G5 zo2X*-tn&u)EJXrX6g9gcr)GHw=H_($Z6mlfv`bTSiinMTs--T8nW(i$t+uH9L$uu= z!)lSSxO2F`mPrY1On1cj4^vw=96UEFoBQYdON`Zi0$S+m5n=A*;$42W{3>PsiYNqMjtrF!1@{7%wPbTqJ=szvEM_I(G*h>pe9$c#%x;2nY^v4gM z@C333$3T~DkawB|aF2-XDf<$b_iU1mI(odd%ktj%VYjE?Ar68ce~rb~cb(qZNLGP2 zjqY+sY^#>XBnL85Ok6zLc$a7kWEMg#DV5RZ-(Mh_Vq1)sZve-Y>$zh%B7AZhtx=s@ z=%A^Pe>UO82UCoT5GJHuSy@?NFzgY`6ju~cW-MFE3TqfYPChm-Ip&
      u%;ww~Hk z0r5!7f@yG2&h#%ELOHZyXpY;g#tHSriER#fsdK?`=8`h4TlE*8#HyP6mfIo3&idau zyfoergU z^qga6^!bgtnpUHU2^}%#qmt&>z<@@W#A3C82dVv9S?iu9bds;zRkuTK<&+4#BV_l% zl{ToTiBPVAY5UbCOo74e3Uvxi1UB+m@*U*^e$8dKK}H8})(*8@s5;90cI_8&J`j_C zljYXA2s=U8IBy%YxkDBmfcB3O+Gj&8Xj7h_cdsEPml|(80iWmh!$`d;OQU> zwjk?=giHK(SC1BLaGjx7X=xL$=D?C4Fn{|g9!Og;Oh63!K&v_`i%$c&(a~W2Z=-~S zUS{x`W-E)_GwN$$N+En2q%xm}Inz8@)|c_hG|5Vv;8&@73Xwbt$jP%ql;s+a@Oc2m zy;03&LW29~`1mp4)JWb9Ul-(GoEefh0|(KtgGT_N(3sixAbq&sHaQES^R0Etr^Y~v zk&&ZrXduim3(kvuG&Hf?^U+=hts8aDe?2VQu=6t6%ku(COrK7rc*SL&V~K=q0#7q*r|G&KFcZjCPvyFQu+O#)V&u=yRbD7!~7%u%Reko zUQ1?(DCIC}11m7odbD7US1%>?8CKgIS~R*7zTd1kQbwRCZBafH=~qR?G5b7@vD1hS z;u-hS{OW?tdZ{#m;KaRRCd#yb-e6f+5e@+jeNF>Ap}}K{qy=#(_XY70pDl8uCOEsg zY7A@wKi;esMrDSrR94*q2H(3gisR|9&;p4+A34*-Yog0}!K7d3a&I~``^n{GwdZ!F z^5RCE9gw7tM(bYv_hZH?9W0_;R?j!(W=c;5zyIy69}a~rEs)+{%Sz6X%E&Xzzs})K zv^BkSLbQecmk~d+CT8Pb5Pmpjt`hg)i=OLn9+28yk6*7Q1d5xE@i@X17^g zVxKdOIi$Xd3?ib05At2xjmA3OYh^ZNf8RJl7fjz0_pq_~q&L&j&J!T$*)&NMf^ozC zpX@8Wr&?Q5yEB3=u3#*0I2}FHiz)Kh-|6Y;<6~Pl0`+`)Fn+N+z>qMgLK{}o-qhN{ zP&~4xzX~t%%4l?64e-D<81OKkr4EbD-1$?#vd6OE;V`wSt8%%R?pv#G!b~U*U;kXt zC*}#Yx~27z&tR^21O^EE!EcQ$PWJ$Wn-m<`N`ve`*^3sNB6`%a?&COfAJXq^;| z-QJaN`b~72HS_TYhkBd8$@LhDbNZGLzff-c?tES<`2rof@OE_qnHAQ=XwA69kS?zI zAZD{h`C6A>yNu6&eY-Con;oNEP`H^eLbLIEMR?S{BFA>HO;?&YUW@5{^kd_C)8&a* zaZGM&wNMLxA|IKBG3wXRryH}a-?)0Oud9EGb0|Y#X}YVd1Cqdq#*hvNcF-OIQ}7&` z48AoDDu6HOsH>wL{@L0ZH5gr3e1{!FM#Gj2Sr=YvNz(#bQ!z7=&~h4L7n?g&UzU@~ z?xj%Q*SO=uw_^>V+G~&!6Z0L}$mI_QrUPEX>&={M)oTA@S}nj1d)wTUz+eIUN|^OS z+lrQoZAdJGVS6Q-gj?CexYZ^JWXKKw@XAfnVHchlw2+_6gy?J&b?vE?H9meW!93{J zM;Vi#gIN}qt)_5wm+=c>vpMQ&Obg{mi+Fl^vK~MJjk!Q#d!#MxB=k(CKIW=BawZG( zFhT{aYI|AO|Kj9px#!XDLblc4a=ZO(v-q!e^qq41-Wl9|45L0wGCQ!J#m4m#l~Iq$ z2nHlQbaZs~_0F_pXBU^msAp3m^kzHxyjtUQVY`O1ia+QL25m#ZoVFMP{RDAa&~ELe zpv$nlSVyel4bUot=LK`TP@n9bNFXO6) zmv>ot-7D$mfA8$r*V9)R!@Kq54P>Q}lc0`m_-X-ShrMY0h+jWyF2C&MlKl9EB+m>Qk zlOVUZ{#L&^5!8|CWa4aC+wXokv_3|BeShaF+ za(*`-Sw5O(0HKwXmI5R%#-T0Xy9o~u-%OG;p&R(EMxP%RF5D^>g-2n7C8NsN(iM75 zfnxGdS^m5cQ2uZ=(HIEx%o1cDL;DEqG+FEPX?AE|7YCsMJGjIF*f-GS0J?vuFShcf zBm&mXE(5`n3`&JGORPXIQ?Tbw6W=v5@p|zf89Qjcpipp=6~8hL{((#%6lcl4D8tg);-}_RWW1EtAxVI zG3n{+m+MpGq=%#XBUc=7fpY3;!@S&~aRttk6BCgXp1__^IFds&Mrh2f#$f@lqc5y4 zFuViEXj#|oCbe;{U#rg5Rmhk}$>v`R`i^H}i6-OWl}je7Nr(Bn4*{>$0J5FFF!Vu6 zqf4;63yuj2gYCioeu;G{3c=XX)5TUu$6?a`vS%XqOV%;N328+a^Gxx`*gC)Pa!G)| zTwP)|q6~gs1_0cRAp}Sz214*g5eaJ4EI~PAfX{OZNC+YkLK?h7rkGj!D((esNkN@4 z`O0KB!4M+E7h(*Amx}^xJ)r0Ww4Pc4&J!covbL@0uB@!A6nzp)>cGFhU&B6Mn0AV7R3Kr(d481d z@sDTm>drmA@BKG54x}$!rm{?hxG2Numuf{Z41fE2fZK`xYsLUkLA@MPQ{8_N7gkZm z$6BS{O7C_Ze$nffJpSp0YXx<};=%JJO@FtW%psk*^#JX@teE6c_JrX&{xlHqax&R?Z z7nfRil0%V%(7?lDS4fxza76j|_)rrg=Etm>1qR6;LO}}Y27glKHv>{dPjK#KUP?Ah zSPSOga|aVLeD@8v3NodB<_4+;9PzFz-&|8_Y{1t`{6S}IH$ zo!mbN#dvm4`osF2Z0e3|G2V;{c8w#biT$0Sj>{7hpUgOreiVGblVj>>2gDD9Z!?(~ z?zu{RP-3B(f9m+u)VG{%Ak06;N+s@_pG8V9Rb*4n`?i{bLh`s!MaDAYueLQ`70ySD zF+X{1C3QA~F)4*8!)N~K(>^d?05?>fYqWTak~;@#5DRidJzp`y~pAP(&4kjs=En_D4kBZ9rh7KEn$t#riWef%~N%7HnLwv7DH z3_^qOXbk+ql75oIw&2iUg<{$};jO%!%XCf?N-C;%3rN{SqvbSt)*a(W0=)`MJFU!* zOp4F&cwu++P|8?D5+;lkOrm$6roO)q0G7`I`IE`1DMCLcOz1l~CY)1qD=WmW;#)Ke z>KVv6W5HhXYRbz}iQE|LZA13>e!lb?qr_PrD?|#d!*bMUbka9lw$U36ZEY-yGM1(@ z$n#8X;h}Jep#{I=|0C5($Ji?a`$Xt6 zNk|M14vN-+J_RBY_xJa4NvhSVa7^Bn>m>9O)6pff3iKdEfT?ggF6_7BE#afVMmSNZ zm5$#R)P&&q6ij16Hp_EiBFa@3wR2`>=IYg}OG`^#%0Sn>By_QAy12L~r(TTm=g*(N zdGki3oN!Vum{1NWH6jU7OD1o@r^@>-%Yq}XSAwsp8a8&R`p{hDSkd_;)HM*)Jhk#k z9#Y|iFhrCX<2ywFPE1TpPfsr_EXXpD>t6KvI6bS_~pp_NgBMt$%9ir$@DDH;f*gi$)a}FJ4?-T@@{KWMri1GtmTewdmjdxSC@8-qKSmX<^XE&3>%dd7Nk zJslhz{Q2jfPo6vx)l@hMgHV8x-GC+`w>1MvLh#e*qI%;C8`O6)f;wk@`g@e_Jz+@5 z(g!^t$wc9Gzj_G3$;n9(fWknLgh)^8CG;i<1wu2S;@aBU)2B~2H#e)*Duf|rpx{3- z&~=~U(b$KiC2d7Rzo4ro2aSBJoqWy@foA!rvPfvEAN-C?D1q07x%7YPVhG60p`oEl zr7|@&wYa#ru&{uUvb~3%B_VKpeEj_R^S}T8`^AeFZ{NNZbre(t4-vI5;@ihLck``h|pm$UtG^ zqeqXPJbALcy)E0qDld4QQL?zC(>Qa! zYqm|KLF=EvUi2ms2yK%!f%Jt68^_1TXJ= zQs>s7cLrZENj4_!#wl5+gB+S{K{K-H2istZnKAIVVd4#Z(a*is(vPtRoed)l2XLpTS&E*&ZDo75Y^wmHGMk%a<<;C;ORx|6GESgaF7UvhL~Ar;v)f zySo@yN|DG~VK9xHB=`8}tEQ&&VlWjhi-fgw|NpE*Xxs&Abj~aRsMD;YQ1f}+cTiK2 z5kI_$p1i}u!@|JX*;!$r&<_FnqAz<#pd_IMT%+qTL@J8iNF7xkkzP|vGHgImZ6fMN zM*NK=^j$cHgobiD;1P7L*7bAh+!vpC&TFznkBiGFdcenYtdGU-6o9OJ9~l`LA0MBd zo)-Fv3W|4&zRn$ll7wNP3<^5x#>R#SM$|*0b&c=}f>C==H4h;kU95KEb>Dg85|QXr zk)_6dP%#*I1PMtS^A(-Q0`QdqHS|h5U^EHII?Y26Nyj7*=wr%@yn-fZ6B84nd5YnO z_ynAyB{WJB<^ejYFzVT}XTrwq?QJm^4-O8Xl+t;l-KTWn=vbl-`UgpSZf;HlAR>W9U(C)$Ny3I8 z^blTwiaR?y`}_MsMKqYAhlXs_6~lh2O|XF2cvD?Ez-Uy^S}5*sNNB7+O+uf6QM!Aw zpZ2Lsjp%gIN!XZ%D^}mo(2zVr`4r8wKSK^DMM=VcAR1vQ>RXI1+i{vD3|%PTt|gvJlTfEt>$Fdj&?RZv zOgdwgtvm{^#Ogzr7q(-9KZb{|3no6tQ_%Hp%dd4;quy4ae@VEzyNia1APg3qc8&aV zf|AJ=rdz2~D@kOa(C_f@@Hhs_qbL}4F1QYFm^4%)2A!7?p2b6#cBmS+r%^jyBGaPK zz$1z~)BF>>LeI}LXU<^!7>LIngmk@fR*w#5a3?`{?)*SJ%uF`qQDkY+i=r6xDG5cB z6wWY3+j$iaWCcm6Get=ltSTL`&?(?-Mr^A1As)3D)S}X;qdDH1C_R9YM;A2tqKtI2 z0k7dFsPP8rS9mPkAM~+zF&jm?7QH~)ioPQux?NEJ)Fh0O zDNb7K=oo`>a=MKAdirUgml_7@gc~CX@z;3Ec>xTU&Z^;R*8A^5%BN0zaVqXNkTlrU zZcm*8%AQMl`U=R2zh?CyInQ}@x|lFZ7uxH~p><;hLPW;)l~nAnjG_1M-;3p@nMP@Y zb>Is_W3ZXXyGx5r`aeN1*gsg=##;j&*w}>Hm*YYtbQ#_nUj@}014(1gTDtALRz9^C z)X&KV>!=8jOE_=3pb2{(#dCPHbfw(ZHeuxDB;Y{`b!3j zJ3+TlP_%0AsG+Xl%$YNlN~P#E(r)w}2_eu(#nib_t`0-RV3i=?)5dQl%un)X^(%g) zUMrbK7sU$JkrEe5;PzR;PdZOh8SOxSjJet*p$l~rHK%dFtGb|+*1jCQ{}6y$N9`Z< z9SMQD_MR+-w{PEKkdvQf=Dane$!TO@F!)ZbOiNnIM&}u>TAUiLx_TsZ>HCY3U`ClV zI{847%NBX%g;`2(r2L$biWYwL)mIY}6Q47>d-!DTAM_~+8v)n7tRYlUxKK#IshVAv z7H&rFX*wCHtJ46m!4=HEfC8+4L@($_wHq{4^p#s*Sn1EZkl)qk9-Tc=+H=FV1_uYn z$H(>I_btuxQIfDB7~Ri#l91c7{!nS*Jh^8yOr!L25&M$RB_sr4g1I$R9l0m|9{M{I zd2xJ0R~{5TbN1}nv9Ynw^(KvH3FHz%lq75kPI^{lvF`5f3Uwnjq7-SF{)!h$?8VTW zi4qs?$jHd(=&1gdyk4zV<@fOcyePjczkiDTEc(>;AW9N;5X`m6U-4NNF6&z-`tcI& zL`lLDB??C0xr0lT=roihEK#Chlq4)sqF|IHEK#Ch{2yDykWdKcYZXhx;AWdO;ATlsAGaxZFIx;poF)$!2FflMNFqEPc00007bV*G` z2j>MA4IBuXb3l~<000SaNLh0L01FZT01FZU(%pXi00004XF*Lt006O%3;baP0007Q zP)t-s|NsB`)ynpnE`^rejLVKx8${{8aq z^qhd~UNHUd>HhZf{ORKS=HBpqS@^)M{qXDi-O=uHO7^dq|NQ#>?B)C3)a`9U_OqS- z`uOvcbnRj{`Np*T=;8LYp6+Ei^O14<Qf`{cvAGEi22jT`qIVmi)!|; zne(EE`N_HY%DVNclJusG?|@wMmU!}uYXAKF@{Miz(ZbwMBH&OWk24m%L>;t29NA4F zkTVv5; zulw1~{P_31hjxoXD~LZRw|#5#=Hlzv(zADDi9aZiNHM~Xe)Ob^@Z#O9ZCRmSN#@VT z-@>?W)H>g2_gg{x~<&!(FD-PFvXlfR35*0H7a zt(KHbHiSAOBqscBVrEEJ<*Ox3TW&ZU>1TSj~_ z8Ln_$&GqvN5B z%19sAbyn}gw7f$d-H~|iZ%62_oz;9`?O!s=WkkP8B<#AX`q#?ymwE4XP3>Sa>|HVK zYC-P5u=v{0^OAD;&A;7nOw~*v?XjKDa8m5Ntntaa)_-I4&%)S-YuAHm#8xuxy{`M? z+Vwy8m;e9(0d!JMQvg8b*k%9#17}G@K~#9!otEcclQ9s-!z%C;sMf+Sse+t1sbQT+6bg zn7QxTy7~rwDWdT$MU4^sYD0ZpEeDwSdr@UH8q?82bqxyi%UAdfc>M-_)>NbQbZA(` z-w~9jWkvxkuK?h|!@|l(kL}!(r-Gc^ZD^j07`I>CBb3=wcnaB_J|lu>&%t5g`3nFP;Gl*(4jahi0&pPj;Gz7(_NX2? zdhGZK_?^q(pq5JjJD6i}R39uk1Zd_`F2e>gOlH=8in22KRSwMHQjSaV)HECDTo6o4 z<&zwU<=4F=7mN+!MV5g(cJA7}XK!5czGNz>#9m_(+_H6BLgI#v+Y{ocpjvpWkU}M1 z#UpUjW^yu=x7FeAajXPfy=HCPy7iLDSo}rS3){!Ij!leUswYc8v@0%}ECKeS6B)Gv z(@JQ%3L8g7N{fzzGPewZ%V`T(+Y&q)kZ4*crF`)c2riu~npJ5ho0f|_+ zh**&{4`jPCYc>(gIoP&3d81?BbB6R7W`^7LlX3QzY}$0n2@0u3YstjePM$(N`>C>c z;zDibL Date: Wed, 22 Jan 2025 16:23:55 -0500 Subject: [PATCH 048/119] WIP --- scenarios/AksOpenAiTerraform/run.sh | 26 +++++ .../script/01-push-app-image.sh | 8 ++ .../04-create-nginx-ingress-controller.sh | 36 ++++++ .../script/05-install-cert-manager.sh | 31 ++++++ .../script/06-create-cluster-issuer.sh | 16 +++ .../07-create-workload-managed-identity.sh | 104 ++++++++++++++++++ .../script/08-create-service-account.sh | 103 +++++++++++++++++ .../script/09-deploy-app.sh | 37 +++++++ .../script/10-create-ingress.sh | 9 ++ .../script/11-configure-dns.sh | 79 +++++++++++++ 10 files changed, 449 insertions(+) create mode 100644 scenarios/AksOpenAiTerraform/script/01-push-app-image.sh create mode 100644 scenarios/AksOpenAiTerraform/script/04-create-nginx-ingress-controller.sh create mode 100644 scenarios/AksOpenAiTerraform/script/05-install-cert-manager.sh create mode 100644 scenarios/AksOpenAiTerraform/script/06-create-cluster-issuer.sh create mode 100644 scenarios/AksOpenAiTerraform/script/07-create-workload-managed-identity.sh create mode 100644 scenarios/AksOpenAiTerraform/script/08-create-service-account.sh create mode 100644 scenarios/AksOpenAiTerraform/script/09-deploy-app.sh create mode 100644 scenarios/AksOpenAiTerraform/script/10-create-ingress.sh create mode 100644 scenarios/AksOpenAiTerraform/script/11-configure-dns.sh diff --git a/scenarios/AksOpenAiTerraform/run.sh b/scenarios/AksOpenAiTerraform/run.sh index adebad18e..b25a2012f 100644 --- a/scenarios/AksOpenAiTerraform/run.sh +++ b/scenarios/AksOpenAiTerraform/run.sh @@ -1,5 +1,31 @@ +export RG_NAME="" + export OPEN_AI_SUBDOMAIN="magic8ball" +# Publish Image +export ACR_NAME=(terraform output -raw acr_name) +export IMAGE="azurecr.io/magic8ball:latest" + +# Nginx Ingress Controller +export nginxNamespace="ingress-basic" +export nginxRepoName="ingress-nginx" +export nginxRepoUrl="https://kubernetes.github.io/ingress-nginx" +export nginxChartName="ingress-nginx" +export nginxReleaseName="nginx-ingress" +export nginxReplicaCount=3 + +# Certificate Manager +export cmNamespace="cert-manager" +export cmRepoName="jetstack" +export cmRepoUrl="https://charts.jetstack.io" +export cmChartName="cert-manager" +export cmReleaseName="cert-manager" + +# Cluster Issuer +email="paolos@microsoft.com" +clusterIssuerName="letsencrypt-nginx" +clusterIssuerTemplate="cluster-issuer.yml" + # Variables acrName="CyanAcr" acrResourceGrougName="CyanRG" diff --git a/scenarios/AksOpenAiTerraform/script/01-push-app-image.sh b/scenarios/AksOpenAiTerraform/script/01-push-app-image.sh new file mode 100644 index 000000000..c0164b0b3 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/script/01-push-app-image.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Login +az acr login --name $ACR_NAME +ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) + +# Build + Push +docker build -t $ACR_URL/$IMAGE ./app --push diff --git a/scenarios/AksOpenAiTerraform/script/04-create-nginx-ingress-controller.sh b/scenarios/AksOpenAiTerraform/script/04-create-nginx-ingress-controller.sh new file mode 100644 index 000000000..f059c37ea --- /dev/null +++ b/scenarios/AksOpenAiTerraform/script/04-create-nginx-ingress-controller.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Use Helm to deploy an NGINX ingress controller +result=$(helm list -n $nginxNamespace | grep $nginxReleaseName | awk '{print $1}') + +if [[ -n $result ]]; then + echo "[$nginxReleaseName] ingress controller already exists in the [$nginxNamespace] namespace" +else + # Check if the ingress-nginx repository is not already added + result=$(helm repo list | grep $nginxRepoName | awk '{print $1}') + + if [[ -n $result ]]; then + echo "[$nginxRepoName] Helm repo already exists" + else + # Add the ingress-nginx repository + echo "Adding [$nginxRepoName] Helm repo..." + helm repo add $nginxRepoName $nginxRepoUrl + fi + + # Update your local Helm chart repository cache + echo 'Updating Helm repos...' + helm repo update + + # Deploy NGINX ingress controller + echo "Deploying [$nginxReleaseName] NGINX ingress controller to the [$nginxNamespace] namespace..." + helm install $nginxReleaseName $nginxRepoName/$nginxChartName \ + --create-namespace \ + --namespace $nginxNamespace \ + --set controller.nodeSelector."kubernetes\.io/os"=linux \ + --set controller.replicaCount=$replicaCount \ + --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz +fi + +# Get values +helm get values $nginxReleaseName --namespace $nginxNamespace diff --git a/scenarios/AksOpenAiTerraform/script/05-install-cert-manager.sh b/scenarios/AksOpenAiTerraform/script/05-install-cert-manager.sh new file mode 100644 index 000000000..3fee03e52 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/script/05-install-cert-manager.sh @@ -0,0 +1,31 @@ +#/bin/bash + +# Check if the ingress-nginx repository is not already added +result=$(helm repo list | grep $cmRepoName | awk '{print $1}') + +if [[ -n $result ]]; then + echo "[$cmRepoName] Helm repo already exists" +else + # Add the Jetstack Helm repository + echo "Adding [$cmRepoName] Helm repo..." + helm repo add $cmRepoName $cmRepoUrl +fi + +# Update your local Helm chart repository cache +echo 'Updating Helm repos...' +helm repo update + +# Install cert-manager Helm chart +result=$(helm list -n $cmNamespace | grep $cmReleaseName | awk '{print $1}') + +if [[ -n $result ]]; then + echo "[$cmReleaseName] cert-manager already exists in the $cmNamespace namespace" +else + # Install the cert-manager Helm chart + echo "Deploying [$cmReleaseName] cert-manager to the $cmNamespace namespace..." + helm install $cmReleaseName $cmRepoName/$cmChartName \ + --create-namespace \ + --namespace $cmNamespace \ + --set installCRDs=true \ + --set nodeSelector."kubernetes\.io/os"=linux +fi diff --git a/scenarios/AksOpenAiTerraform/script/06-create-cluster-issuer.sh b/scenarios/AksOpenAiTerraform/script/06-create-cluster-issuer.sh new file mode 100644 index 000000000..9ab805a54 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/script/06-create-cluster-issuer.sh @@ -0,0 +1,16 @@ +#/bin/bash + +# Check if the cluster issuer already exists +result=$(kubectl get ClusterIssuer -o json | jq -r '.items[].metadata.name | select(. == "'$clusterIssuerName'")') + +if [[ -n $result ]]; then + echo "[$clusterIssuerName] cluster issuer already exists" + exit +else + # Create the cluster issuer + echo "[$clusterIssuerName] cluster issuer does not exist" + echo "Creating [$clusterIssuerName] cluster issuer..." + cat $clusterIssuerTemplate | + yq "(.spec.acme.email)|="\""$email"\" | + kubectl apply -f - +fi diff --git a/scenarios/AksOpenAiTerraform/script/07-create-workload-managed-identity.sh b/scenarios/AksOpenAiTerraform/script/07-create-workload-managed-identity.sh new file mode 100644 index 000000000..c770e6476 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/script/07-create-workload-managed-identity.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# Variables +source ./00-variables.sh + +# Check if the user-assigned managed identity already exists +echo "Checking if [$managedIdentityName] user-assigned managed identity actually exists in the [$aksResourceGroupName] resource group..." + +az identity show \ + --name $managedIdentityName \ + --resource-group $aksResourceGroupName &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$managedIdentityName] user-assigned managed identity actually exists in the [$aksResourceGroupName] resource group" + echo "Creating [$managedIdentityName] user-assigned managed identity in the [$aksResourceGroupName] resource group..." + + # Create the user-assigned managed identity + az identity create \ + --name $managedIdentityName \ + --resource-group $aksResourceGroupName \ + --location $location \ + --subscription $subscriptionId 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$managedIdentityName] user-assigned managed identity successfully created in the [$aksResourceGroupName] resource group" + else + echo "Failed to create [$managedIdentityName] user-assigned managed identity in the [$aksResourceGroupName] resource group" + exit + fi +else + echo "[$managedIdentityName] user-assigned managed identity already exists in the [$aksResourceGroupName] resource group" +fi + +# Retrieve the clientId of the user-assigned managed identity +echo "Retrieving clientId for [$managedIdentityName] managed identity..." +clientId=$(az identity show \ + --name $managedIdentityName \ + --resource-group $aksResourceGroupName \ + --query clientId \ + --output tsv) + +if [[ -n $clientId ]]; then + echo "[$clientId] clientId for the [$managedIdentityName] managed identity successfully retrieved" +else + echo "Failed to retrieve clientId for the [$managedIdentityName] managed identity" + exit +fi + +# Retrieve the principalId of the user-assigned managed identity +echo "Retrieving principalId for [$managedIdentityName] managed identity..." +principalId=$(az identity show \ + --name $managedIdentityName \ + --resource-group $aksResourceGroupName \ + --query principalId \ + --output tsv) + +if [[ -n $principalId ]]; then + echo "[$principalId] principalId for the [$managedIdentityName] managed identity successfully retrieved" +else + echo "Failed to retrieve principalId for the [$managedIdentityName] managed identity" + exit +fi + +# Get the resource id of the Azure OpenAI resource +openAiId=$(az cognitiveservices account show \ + --name $openAiName \ + --resource-group $openAiResourceGroupName \ + --query id \ + --output tsv) + +if [[ -n $openAiId ]]; then + echo "Resource id for the [$openAiName] Azure OpenAI resource successfully retrieved" +else + echo "Failed to the resource id for the [$openAiName] Azure OpenAI resource" + exit -1 +fi + +# Assign the Cognitive Services User role on the Azure OpenAI resource to the managed identity +role="Cognitive Services User" +echo "Checking if the [$managedIdentityName] managed identity has been assigned to [$role] role with [$openAiName] Azure OpenAI resource as a scope..." +current=$(az role assignment list \ + --assignee $principalId \ + --scope $openAiId \ + --query "[?roleDefinitionName=='$role'].roleDefinitionName" \ + --output tsv 2>/dev/null) + +if [[ $current == $role ]]; then + echo "[$managedIdentityName] managed identity is already assigned to the ["$current"] role with [$openAiName] Azure OpenAI resource as a scope" +else + echo "[$managedIdentityName] managed identity is not assigned to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" + echo "Assigning the [$role] role to the [$managedIdentityName] managed identity with [$openAiName] Azure OpenAI resource as a scope..." + + az role assignment create \ + --assignee $principalId \ + --role "$role" \ + --scope $openAiId 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$managedIdentityName] managed identity successfully assigned to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" + else + echo "Failed to assign the [$managedIdentityName] managed identity to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" + exit + fi +fi \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/script/08-create-service-account.sh b/scenarios/AksOpenAiTerraform/script/08-create-service-account.sh new file mode 100644 index 000000000..5a89a0619 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/script/08-create-service-account.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +# Variables for the user-assigned managed identity +source ./00-variables.sh + +# Check if the namespace already exists +result=$(kubectl get namespace -o 'jsonpath={.items[?(@.metadata.name=="'$namespace'")].metadata.name'}) + +if [[ -n $result ]]; then + echo "[$namespace] namespace already exists" +else + # Create the namespace for your ingress resources + echo "[$namespace] namespace does not exist" + echo "Creating [$namespace] namespace..." + kubectl create namespace $namespace +fi + +# Check if the service account already exists +result=$(kubectl get sa -n $namespace -o 'jsonpath={.items[?(@.metadata.name=="'$serviceAccountName'")].metadata.name'}) + +if [[ -n $result ]]; then + echo "[$serviceAccountName] service account already exists" +else + # Retrieve the resource id of the user-assigned managed identity + echo "Retrieving clientId for [$managedIdentityName] managed identity..." + managedIdentityClientId=$(az identity show \ + --name $managedIdentityName \ + --resource-group $aksResourceGroupName \ + --query clientId \ + --output tsv) + + if [[ -n $managedIdentityClientId ]]; then + echo "[$managedIdentityClientId] clientId for the [$managedIdentityName] managed identity successfully retrieved" + else + echo "Failed to retrieve clientId for the [$managedIdentityName] managed identity" + exit + fi + + # Create the service account + echo "[$serviceAccountName] service account does not exist" + echo "Creating [$serviceAccountName] service account..." + cat </dev/null + +if [[ $? != 0 ]]; then + echo "No [$federatedIdentityName] federated identity credential actually exists in the [$aksResourceGroupName] resource group" + + # Get the OIDC Issuer URL + aksOidcIssuerUrl="$(az aks show \ + --only-show-errors \ + --name $aksClusterName \ + --resource-group $aksResourceGroupName \ + --query oidcIssuerProfile.issuerUrl \ + --output tsv)" + + # Show OIDC Issuer URL + if [[ -n $aksOidcIssuerUrl ]]; then + echo "The OIDC Issuer URL of the $aksClusterName cluster is $aksOidcIssuerUrl" + fi + + echo "Creating [$federatedIdentityName] federated identity credential in the [$aksResourceGroupName] resource group..." + + # Establish the federated identity credential between the managed identity, the service account issuer, and the subject. + az identity federated-credential create \ + --name $federatedIdentityName \ + --identity-name $managedIdentityName \ + --resource-group $aksResourceGroupName \ + --issuer $aksOidcIssuerUrl \ + --subject system:serviceaccount:$namespace:$serviceAccountName + + if [[ $? == 0 ]]; then + echo "[$federatedIdentityName] federated identity credential successfully created in the [$aksResourceGroupName] resource group" + else + echo "Failed to create [$federatedIdentityName] federated identity credential in the [$aksResourceGroupName] resource group" + exit + fi +else + echo "[$federatedIdentityName] federated identity credential already exists in the [$aksResourceGroupName] resource group" +fi \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/script/09-deploy-app.sh b/scenarios/AksOpenAiTerraform/script/09-deploy-app.sh new file mode 100644 index 000000000..f9e1d757c --- /dev/null +++ b/scenarios/AksOpenAiTerraform/script/09-deploy-app.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Variables +source ./00-variables.sh + +# Check if namespace exists in the cluster +result=$(kubectl get namespace -o jsonpath="{.items[?(@.metadata.name=='$namespace')].metadata.name}") + +if [[ -n $result ]]; then + echo "$namespace namespace already exists in the cluster" +else + echo "$namespace namespace does not exist in the cluster" + echo "creating $namespace namespace in the cluster..." + kubectl create namespace $namespace +fi + +# Create config map +cat $configMapTemplate | + yq "(.data.TITLE)|="\""$title"\" | + yq "(.data.LABEL)|="\""$label"\" | + yq "(.data.TEMPERATURE)|="\""$temperature"\" | + yq "(.data.IMAGE_WIDTH)|="\""$imageWidth"\" | + yq "(.data.AZURE_OPENAI_TYPE)|="\""$openAiType"\" | + yq "(.data.AZURE_OPENAI_BASE)|="\""$openAiBase"\" | + yq "(.data.AZURE_OPENAI_MODEL)|="\""$openAiModel"\" | + yq "(.data.AZURE_OPENAI_DEPLOYMENT)|="\""$openAiDeployment"\" | + kubectl apply -n $namespace -f - + +# Create deployment +cat $deploymentTemplate | + yq "(.spec.template.spec.containers[0].image)|="\""$image"\" | + yq "(.spec.template.spec.containers[0].imagePullPolicy)|="\""$imagePullPolicy"\" | + yq "(.spec.template.spec.serviceAccountName)|="\""$serviceAccountName"\" | + kubectl apply -n $namespace -f - + +# Create deployment +kubectl apply -f $serviceTemplate -n $namespace \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/script/10-create-ingress.sh b/scenarios/AksOpenAiTerraform/script/10-create-ingress.sh new file mode 100644 index 000000000..52f090706 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/script/10-create-ingress.sh @@ -0,0 +1,9 @@ +#/bin/bash + +# Create the ingress +echo "[$ingressName] ingress does not exist" +echo "Creating [$ingressName] ingress..." +cat $ingressTemplate | + yq "(.spec.tls[0].hosts[0])|="\""$host"\" | + yq "(.spec.rules[0].host)|="\""$host"\" | + kubectl apply -n $namespace -f - \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/script/11-configure-dns.sh b/scenarios/AksOpenAiTerraform/script/11-configure-dns.sh new file mode 100644 index 000000000..95f8baf69 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/script/11-configure-dns.sh @@ -0,0 +1,79 @@ +# Variables +source ./00-variables.sh + +# Retrieve the public IP address from the ingress +echo "Retrieving the external IP address from the [$ingressName] ingress..." +publicIpAddress=$(kubectl get ingress $ingressName -n $namespace -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + +if [ -n $publicIpAddress ]; then + echo "[$publicIpAddress] external IP address of the application gateway ingress controller successfully retrieved from the [$ingressName] ingress" +else + echo "Failed to retrieve the external IP address of the application gateway ingress controller from the [$ingressName] ingress" + exit +fi + +# Check if an A record for todolist subdomain exists in the DNS Zone +echo "Retrieving the A record for the [$subdomain] subdomain from the [$dnsZoneName] DNS zone..." +ipv4Address=$(az network dns record-set a list \ + --zone-name $dnsZoneName \ + --resource-group $dnsZoneResourceGroupName \ + --query "[?name=='$subdomain'].arecords[].ipv4Address" \ + --output tsv) + +if [[ -n $ipv4Address ]]; then + echo "An A record already exists in [$dnsZoneName] DNS zone for the [$subdomain] subdomain with [$ipv4Address] IP address" + + if [[ $ipv4Address == $publicIpAddress ]]; then + echo "The [$ipv4Address] ip address of the existing A record is equal to the ip address of the [$ingressName] ingress" + echo "No additional step is required" + exit + else + echo "The [$ipv4Address] ip address of the existing A record is different than the ip address of the [$ingressName] ingress" + fi + + # Retrieving name of the record set relative to the zone + echo "Retrieving the name of the record set relative to the [$dnsZoneName] zone..." + + recordSetName=$(az network dns record-set a list \ + --zone-name $dnsZoneName \ + --resource-group $dnsZoneResourceGroupName \ + --query "[?name=='$subdomain'].name" \ + --output name 2>/dev/null) + + if [[ -n $recordSetName ]]; then + "[$recordSetName] record set name successfully retrieved" + else + "Failed to retrieve the name of the record set relative to the [$dnsZoneName] zone" + exit + fi + + # Remove the a record + echo "Removing the A record from the record set relative to the [$dnsZoneName] zone..." + + az network dns record-set a remove-record \ + --ipv4-address $ipv4Address \ + --record-set-name $recordSetName \ + --zone-name $dnsZoneName \ + --resource-group $dnsZoneResourceGroupName + + if [[ $? == 0 ]]; then + echo "[$ipv4Address] ip address successfully removed from the [$recordSetName] record set" + else + echo "Failed to remove the [$ipv4Address] ip address from the [$recordSetName] record set" + exit + fi +fi + +# Create the a record +echo "Creating an A record in [$dnsZoneName] DNS zone for the [$subdomain] subdomain with [$publicIpAddress] IP address..." +az network dns record-set a add-record \ + --zone-name $dnsZoneName \ + --resource-group $dnsZoneResourceGroupName \ + --record-set-name $subdomain \ + --ipv4-address $publicIpAddress 1>/dev/null + +if [[ $? == 0 ]]; then + echo "A record for the [$subdomain] subdomain with [$publicIpAddress] IP address successfully created in [$dnsZoneName] DNS zone" +else + echo "Failed to create an A record for the $subdomain subdomain with [$publicIpAddress] IP address in [$dnsZoneName] DNS zone" +fi From 96da54f1dedfc64c9c95c910e77dbc3011bdcae7 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 22 Jan 2025 21:19:16 -0500 Subject: [PATCH 049/119] Rename --- .../{script => wip}/01-push-app-image.sh | 0 .../04-create-nginx-ingress-controller.sh | 0 .../{script => wip}/05-install-cert-manager.sh | 0 .../{script => wip}/06-create-cluster-issuer.sh | 0 .../07-create-workload-managed-identity.sh | 0 .../{script => wip}/08-create-service-account.sh | 0 .../{script => wip}/09-deploy-app.sh | 0 .../{script => wip}/10-create-ingress.sh | 0 .../{script => wip}/11-configure-dns.sh | 0 .../{script => wip}/app/Dockerfile | 0 .../AksOpenAiTerraform/{script => wip}/app/app.py | 0 .../{script => wip}/app/images/magic8ball.png | Bin .../{script => wip}/app/images/robot.png | Bin .../{script => wip}/app/requirements.txt | 0 .../install-nginx-via-helm-and-create-sa.sh | 0 .../{script => wip}/manifests/cluster-issuer.yml | 0 .../{script => wip}/manifests/configMap.yml | 0 .../{script => wip}/manifests/deployment.yml | 0 .../{script => wip}/manifests/ingress.yml | 0 .../{script => wip}/manifests/service.yml | 0 20 files changed, 0 insertions(+), 0 deletions(-) rename scenarios/AksOpenAiTerraform/{script => wip}/01-push-app-image.sh (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/04-create-nginx-ingress-controller.sh (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/05-install-cert-manager.sh (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/06-create-cluster-issuer.sh (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/07-create-workload-managed-identity.sh (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/08-create-service-account.sh (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/09-deploy-app.sh (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/10-create-ingress.sh (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/11-configure-dns.sh (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/app/Dockerfile (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/app/app.py (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/app/images/magic8ball.png (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/app/images/robot.png (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/app/requirements.txt (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/install-nginx-via-helm-and-create-sa.sh (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/manifests/cluster-issuer.yml (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/manifests/configMap.yml (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/manifests/deployment.yml (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/manifests/ingress.yml (100%) rename scenarios/AksOpenAiTerraform/{script => wip}/manifests/service.yml (100%) diff --git a/scenarios/AksOpenAiTerraform/script/01-push-app-image.sh b/scenarios/AksOpenAiTerraform/wip/01-push-app-image.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/script/01-push-app-image.sh rename to scenarios/AksOpenAiTerraform/wip/01-push-app-image.sh diff --git a/scenarios/AksOpenAiTerraform/script/04-create-nginx-ingress-controller.sh b/scenarios/AksOpenAiTerraform/wip/04-create-nginx-ingress-controller.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/script/04-create-nginx-ingress-controller.sh rename to scenarios/AksOpenAiTerraform/wip/04-create-nginx-ingress-controller.sh diff --git a/scenarios/AksOpenAiTerraform/script/05-install-cert-manager.sh b/scenarios/AksOpenAiTerraform/wip/05-install-cert-manager.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/script/05-install-cert-manager.sh rename to scenarios/AksOpenAiTerraform/wip/05-install-cert-manager.sh diff --git a/scenarios/AksOpenAiTerraform/script/06-create-cluster-issuer.sh b/scenarios/AksOpenAiTerraform/wip/06-create-cluster-issuer.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/script/06-create-cluster-issuer.sh rename to scenarios/AksOpenAiTerraform/wip/06-create-cluster-issuer.sh diff --git a/scenarios/AksOpenAiTerraform/script/07-create-workload-managed-identity.sh b/scenarios/AksOpenAiTerraform/wip/07-create-workload-managed-identity.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/script/07-create-workload-managed-identity.sh rename to scenarios/AksOpenAiTerraform/wip/07-create-workload-managed-identity.sh diff --git a/scenarios/AksOpenAiTerraform/script/08-create-service-account.sh b/scenarios/AksOpenAiTerraform/wip/08-create-service-account.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/script/08-create-service-account.sh rename to scenarios/AksOpenAiTerraform/wip/08-create-service-account.sh diff --git a/scenarios/AksOpenAiTerraform/script/09-deploy-app.sh b/scenarios/AksOpenAiTerraform/wip/09-deploy-app.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/script/09-deploy-app.sh rename to scenarios/AksOpenAiTerraform/wip/09-deploy-app.sh diff --git a/scenarios/AksOpenAiTerraform/script/10-create-ingress.sh b/scenarios/AksOpenAiTerraform/wip/10-create-ingress.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/script/10-create-ingress.sh rename to scenarios/AksOpenAiTerraform/wip/10-create-ingress.sh diff --git a/scenarios/AksOpenAiTerraform/script/11-configure-dns.sh b/scenarios/AksOpenAiTerraform/wip/11-configure-dns.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/script/11-configure-dns.sh rename to scenarios/AksOpenAiTerraform/wip/11-configure-dns.sh diff --git a/scenarios/AksOpenAiTerraform/script/app/Dockerfile b/scenarios/AksOpenAiTerraform/wip/app/Dockerfile similarity index 100% rename from scenarios/AksOpenAiTerraform/script/app/Dockerfile rename to scenarios/AksOpenAiTerraform/wip/app/Dockerfile diff --git a/scenarios/AksOpenAiTerraform/script/app/app.py b/scenarios/AksOpenAiTerraform/wip/app/app.py similarity index 100% rename from scenarios/AksOpenAiTerraform/script/app/app.py rename to scenarios/AksOpenAiTerraform/wip/app/app.py diff --git a/scenarios/AksOpenAiTerraform/script/app/images/magic8ball.png b/scenarios/AksOpenAiTerraform/wip/app/images/magic8ball.png similarity index 100% rename from scenarios/AksOpenAiTerraform/script/app/images/magic8ball.png rename to scenarios/AksOpenAiTerraform/wip/app/images/magic8ball.png diff --git a/scenarios/AksOpenAiTerraform/script/app/images/robot.png b/scenarios/AksOpenAiTerraform/wip/app/images/robot.png similarity index 100% rename from scenarios/AksOpenAiTerraform/script/app/images/robot.png rename to scenarios/AksOpenAiTerraform/wip/app/images/robot.png diff --git a/scenarios/AksOpenAiTerraform/script/app/requirements.txt b/scenarios/AksOpenAiTerraform/wip/app/requirements.txt similarity index 100% rename from scenarios/AksOpenAiTerraform/script/app/requirements.txt rename to scenarios/AksOpenAiTerraform/wip/app/requirements.txt diff --git a/scenarios/AksOpenAiTerraform/script/install-nginx-via-helm-and-create-sa.sh b/scenarios/AksOpenAiTerraform/wip/install-nginx-via-helm-and-create-sa.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/script/install-nginx-via-helm-and-create-sa.sh rename to scenarios/AksOpenAiTerraform/wip/install-nginx-via-helm-and-create-sa.sh diff --git a/scenarios/AksOpenAiTerraform/script/manifests/cluster-issuer.yml b/scenarios/AksOpenAiTerraform/wip/manifests/cluster-issuer.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/script/manifests/cluster-issuer.yml rename to scenarios/AksOpenAiTerraform/wip/manifests/cluster-issuer.yml diff --git a/scenarios/AksOpenAiTerraform/script/manifests/configMap.yml b/scenarios/AksOpenAiTerraform/wip/manifests/configMap.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/script/manifests/configMap.yml rename to scenarios/AksOpenAiTerraform/wip/manifests/configMap.yml diff --git a/scenarios/AksOpenAiTerraform/script/manifests/deployment.yml b/scenarios/AksOpenAiTerraform/wip/manifests/deployment.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/script/manifests/deployment.yml rename to scenarios/AksOpenAiTerraform/wip/manifests/deployment.yml diff --git a/scenarios/AksOpenAiTerraform/script/manifests/ingress.yml b/scenarios/AksOpenAiTerraform/wip/manifests/ingress.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/script/manifests/ingress.yml rename to scenarios/AksOpenAiTerraform/wip/manifests/ingress.yml diff --git a/scenarios/AksOpenAiTerraform/script/manifests/service.yml b/scenarios/AksOpenAiTerraform/wip/manifests/service.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/script/manifests/service.yml rename to scenarios/AksOpenAiTerraform/wip/manifests/service.yml From 54c1712900c31c9e6fd461f3a3f513edfc26eb40 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 22 Jan 2025 21:19:24 -0500 Subject: [PATCH 050/119] Add plan --- scenarios/AksOpenAiTerraform/plan.txt | 1116 +++++++++++++++++++++++++ 1 file changed, 1116 insertions(+) create mode 100644 scenarios/AksOpenAiTerraform/plan.txt diff --git a/scenarios/AksOpenAiTerraform/plan.txt b/scenarios/AksOpenAiTerraform/plan.txt new file mode 100644 index 000000000..aa17b1c49 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/plan.txt @@ -0,0 +1,1116 @@ +Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the +following symbols: + + create + +Terraform will perform the following actions: + + # azurerm_federated_identity_credential.federated_identity_credential will be created + + resource "azurerm_federated_identity_credential" "federated_identity_credential" { + + audience = [ + + "api://AzureADTokenExchange", + ] + + id = (known after apply) + + issuer = (known after apply) + + name = "Magic8ballFederatedIdentity" + + parent_id = (known after apply) + + resource_group_name = (known after apply) + + subject = "system:serviceaccount:magic8ball:magic8ball-sa" + } + + # azurerm_resource_group.rg will be created + + resource "azurerm_resource_group" "rg" { + + id = (known after apply) + + location = "westus3" + + name = (known after apply) + } + + # azurerm_role_assignment.acr_pull_assignment will be created + + resource "azurerm_role_assignment" "acr_pull_assignment" { + + condition_version = (known after apply) + + id = (known after apply) + + name = (known after apply) + + principal_id = (known after apply) + + principal_type = (known after apply) + + role_definition_id = (known after apply) + + role_definition_name = "AcrPull" + + scope = (known after apply) + + skip_service_principal_aad_check = (known after apply) + } + + # azurerm_role_assignment.cognitive_services_user_assignment will be created + + resource "azurerm_role_assignment" "cognitive_services_user_assignment" { + + condition_version = (known after apply) + + id = (known after apply) + + name = (known after apply) + + principal_id = (known after apply) + + principal_type = (known after apply) + + role_definition_id = (known after apply) + + role_definition_name = "Cognitive Services User" + + scope = (known after apply) + + skip_service_principal_aad_check = (known after apply) + } + + # azurerm_role_assignment.network_contributor_assignment will be created + + resource "azurerm_role_assignment" "network_contributor_assignment" { + + condition_version = (known after apply) + + id = (known after apply) + + name = (known after apply) + + principal_id = (known after apply) + + principal_type = (known after apply) + + role_definition_id = (known after apply) + + role_definition_name = "Network Contributor" + + scope = (known after apply) + + skip_service_principal_aad_check = (known after apply) + } + + # azurerm_user_assigned_identity.aks_workload_identity will be created + + resource "azurerm_user_assigned_identity" "aks_workload_identity" { + + client_id = (known after apply) + + id = (known after apply) + + location = "westus3" + + name = "WorkloadManagedIdentity" + + principal_id = (known after apply) + + resource_group_name = (known after apply) + + tenant_id = (known after apply) + } + + # random_string.rg_suffix will be created + + resource "random_string" "rg_suffix" { + + id = (known after apply) + + length = 6 + + lower = false + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + numeric = true + + result = (known after apply) + + special = false + + upper = false + } + + # random_string.storage_account_suffix will be created + + resource "random_string" "storage_account_suffix" { + + id = (known after apply) + + length = 8 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = false + + numeric = false + + result = (known after apply) + + special = false + + upper = false + } + + # module.acr_private_dns_zone.azurerm_private_dns_zone.private_dns_zone will be created + + resource "azurerm_private_dns_zone" "private_dns_zone" { + + id = (known after apply) + + max_number_of_record_sets = (known after apply) + + max_number_of_virtual_network_links = (known after apply) + + max_number_of_virtual_network_links_with_registration = (known after apply) + + name = "privatelink.azurecr.io" + + number_of_record_sets = (known after apply) + + resource_group_name = (known after apply) + + + soa_record (known after apply) + } + + # module.acr_private_dns_zone.azurerm_private_dns_zone_virtual_network_link.link["AksVNet"] will be created + + resource "azurerm_private_dns_zone_virtual_network_link" "link" { + + id = (known after apply) + + name = "link_to_aksvnet" + + private_dns_zone_name = "privatelink.azurecr.io" + + registration_enabled = false + + resource_group_name = (known after apply) + + virtual_network_id = (known after apply) + } + + # module.acr_private_endpoint.azurerm_private_endpoint.private_endpoint will be created + + resource "azurerm_private_endpoint" "private_endpoint" { + + custom_dns_configs = (known after apply) + + id = (known after apply) + + location = "westus3" + + name = "AcrPrivateEndpoint" + + network_interface = (known after apply) + + private_dns_zone_configs = (known after apply) + + resource_group_name = (known after apply) + + subnet_id = (known after apply) + + + private_dns_zone_group { + + id = (known after apply) + + name = "AcrPrivateDnsZoneGroup" + + private_dns_zone_ids = (known after apply) + } + + + private_service_connection { + + is_manual_connection = false + + name = "AcrPrivateEndpointConnection" + + private_connection_resource_id = (known after apply) + + private_ip_address = (known after apply) + + subresource_names = [ + + "registry", + ] + } + } + + # module.aks_cluster.azurerm_kubernetes_cluster.aks_cluster will be created + + resource "azurerm_kubernetes_cluster" "aks_cluster" { + + automatic_upgrade_channel = "stable" + + azure_policy_enabled = true + + current_kubernetes_version = (known after apply) + + dns_prefix = "akscluster" + + fqdn = (known after apply) + + http_application_routing_enabled = false + + http_application_routing_zone_name = (known after apply) + + id = (known after apply) + + image_cleaner_enabled = true + + image_cleaner_interval_hours = 72 + + kube_admin_config = (sensitive value) + + kube_admin_config_raw = (sensitive value) + + kube_config = (sensitive value) + + kube_config_raw = (sensitive value) + + kubernetes_version = "1.30.7" + + location = "westus3" + + name = "AksCluster" + + node_os_upgrade_channel = "NodeImage" + + node_resource_group = (known after apply) + + node_resource_group_id = (known after apply) + + oidc_issuer_enabled = true + + oidc_issuer_url = (known after apply) + + open_service_mesh_enabled = true + + portal_fqdn = (known after apply) + + private_cluster_enabled = false + + private_cluster_public_fqdn_enabled = false + + private_dns_zone_id = (known after apply) + + private_fqdn = (known after apply) + + resource_group_name = (known after apply) + + role_based_access_control_enabled = true + + run_command_enabled = true + + sku_tier = "Free" + + support_plan = "KubernetesOfficial" + + workload_identity_enabled = true + + + auto_scaler_profile (known after apply) + + + azure_active_directory_role_based_access_control { + + azure_rbac_enabled = true + + tenant_id = "72f988bf-86f1-41af-91ab-2d7cd011db47" + } + + + default_node_pool { + + kubelet_disk_type = (known after apply) + + max_pods = 50 + + name = "system" + + node_count = 1 + + node_labels = (known after apply) + + orchestrator_version = (known after apply) + + os_disk_size_gb = (known after apply) + + os_disk_type = "Ephemeral" + + os_sku = (known after apply) + + pod_subnet_id = (known after apply) + + scale_down_mode = "Delete" + + type = "VirtualMachineScaleSets" + + ultra_ssd_enabled = false + + vm_size = "Standard_D8ds_v5" + + vnet_subnet_id = (known after apply) + + workload_runtime = (known after apply) + + zones = [ + + "1", + + "2", + + "3", + ] + + + upgrade_settings { + + drain_timeout_in_minutes = 0 + + max_surge = "10%" + + node_soak_duration_in_minutes = 0 + } + } + + + identity { + + identity_ids = (known after apply) + + principal_id = (known after apply) + + tenant_id = (known after apply) + + type = "UserAssigned" + } + + + kubelet_identity (known after apply) + + + network_profile { + + dns_service_ip = "10.2.0.10" + + ip_versions = (known after apply) + + load_balancer_sku = "standard" + + network_data_plane = "azure" + + network_mode = (known after apply) + + network_plugin = "azure" + + network_policy = (known after apply) + + outbound_type = "userAssignedNATGateway" + + pod_cidr = (known after apply) + + pod_cidrs = (known after apply) + + service_cidr = "10.2.0.0/24" + + service_cidrs = (known after apply) + + + load_balancer_profile (known after apply) + + + nat_gateway_profile (known after apply) + } + + + oms_agent { + + log_analytics_workspace_id = (known after apply) + + msi_auth_for_monitoring_enabled = true + + oms_agent_identity = (known after apply) + } + + + windows_profile (known after apply) + + + workload_autoscaler_profile { + + keda_enabled = true + + vertical_pod_autoscaler_enabled = true + } + } + + # module.aks_cluster.azurerm_kubernetes_cluster_node_pool.node_pool will be created + + resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { + + id = (known after apply) + + kubelet_disk_type = (known after apply) + + kubernetes_cluster_id = (known after apply) + + max_pods = 50 + + mode = "User" + + name = "user" + + node_count = (known after apply) + + node_labels = (known after apply) + + orchestrator_version = "1.30.7" + + os_disk_size_gb = (known after apply) + + os_disk_type = "Ephemeral" + + os_sku = (known after apply) + + os_type = "Linux" + + pod_subnet_id = (known after apply) + + priority = "Regular" + + scale_down_mode = "Delete" + + spot_max_price = -1 + + ultra_ssd_enabled = false + + vm_size = "Standard_D8ds_v5" + + vnet_subnet_id = (known after apply) + + zones = [ + + "1", + + "2", + + "3", + ] + } + + # module.aks_cluster.azurerm_monitor_diagnostic_setting.settings will be created + + resource "azurerm_monitor_diagnostic_setting" "settings" { + + id = (known after apply) + + log_analytics_destination_type = (known after apply) + + log_analytics_workspace_id = (known after apply) + + name = "AksDiagnosticsSettings" + + target_resource_id = (known after apply) + + + enabled_log { + + category = "cluster-autoscaler" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "guard" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "kube-apiserver" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "kube-audit" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "kube-audit-admin" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "kube-controller-manager" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "kube-scheduler" + # (1 unchanged attribute hidden) + } + + + metric { + + category = "AllMetrics" + + enabled = true + } + } + + # module.aks_cluster.azurerm_user_assigned_identity.aks_identity will be created + + resource "azurerm_user_assigned_identity" "aks_identity" { + + client_id = (known after apply) + + id = (known after apply) + + location = "westus3" + + name = "AksClusterIdentity" + + principal_id = (known after apply) + + resource_group_name = (known after apply) + + tenant_id = (known after apply) + } + + # module.bastion_host.azurerm_bastion_host.bastion_host will be created + + resource "azurerm_bastion_host" "bastion_host" { + + copy_paste_enabled = true + + dns_name = (known after apply) + + file_copy_enabled = false + + id = (known after apply) + + ip_connect_enabled = false + + kerberos_enabled = false + + location = "westus3" + + name = "BastionHost" + + resource_group_name = (known after apply) + + scale_units = 2 + + session_recording_enabled = false + + shareable_link_enabled = false + + sku = "Basic" + + tunneling_enabled = false + + + ip_configuration { + + name = "configuration" + + public_ip_address_id = (known after apply) + + subnet_id = (known after apply) + } + } + + # module.bastion_host.azurerm_monitor_diagnostic_setting.pip_settings will be created + + resource "azurerm_monitor_diagnostic_setting" "pip_settings" { + + id = (known after apply) + + log_analytics_destination_type = (known after apply) + + log_analytics_workspace_id = (known after apply) + + name = "BastionDdosDiagnosticsSettings" + + target_resource_id = (known after apply) + + + enabled_log { + + category = "DDoSMitigationFlowLogs" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "DDoSMitigationReports" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "DDoSProtectionNotifications" + # (1 unchanged attribute hidden) + } + + + metric { + + category = "AllMetrics" + + enabled = true + } + } + + # module.bastion_host.azurerm_monitor_diagnostic_setting.settings will be created + + resource "azurerm_monitor_diagnostic_setting" "settings" { + + id = (known after apply) + + log_analytics_destination_type = (known after apply) + + log_analytics_workspace_id = (known after apply) + + name = "BastionDiagnosticsSettings" + + target_resource_id = (known after apply) + + + enabled_log { + + category = "BastionAuditLogs" + # (1 unchanged attribute hidden) + } + + + metric { + + category = "AllMetrics" + + enabled = true + } + } + + # module.bastion_host.azurerm_public_ip.public_ip will be created + + resource "azurerm_public_ip" "public_ip" { + + allocation_method = "Static" + + ddos_protection_mode = "VirtualNetworkInherited" + + fqdn = (known after apply) + + id = (known after apply) + + idle_timeout_in_minutes = 4 + + ip_address = (known after apply) + + ip_version = "IPv4" + + location = "westus3" + + name = "BastionHostPublicIp" + + resource_group_name = (known after apply) + + sku = "Standard" + + sku_tier = "Regional" + } + + # module.blob_private_dns_zone.azurerm_private_dns_zone.private_dns_zone will be created + + resource "azurerm_private_dns_zone" "private_dns_zone" { + + id = (known after apply) + + max_number_of_record_sets = (known after apply) + + max_number_of_virtual_network_links = (known after apply) + + max_number_of_virtual_network_links_with_registration = (known after apply) + + name = "privatelink.blob.core.windows.net" + + number_of_record_sets = (known after apply) + + resource_group_name = (known after apply) + + + soa_record (known after apply) + } + + # module.blob_private_dns_zone.azurerm_private_dns_zone_virtual_network_link.link["AksVNet"] will be created + + resource "azurerm_private_dns_zone_virtual_network_link" "link" { + + id = (known after apply) + + name = "link_to_aksvnet" + + private_dns_zone_name = "privatelink.blob.core.windows.net" + + registration_enabled = false + + resource_group_name = (known after apply) + + virtual_network_id = (known after apply) + } + + # module.blob_private_endpoint.azurerm_private_endpoint.private_endpoint will be created + + resource "azurerm_private_endpoint" "private_endpoint" { + + custom_dns_configs = (known after apply) + + id = (known after apply) + + location = "westus3" + + name = "BlobStoragePrivateEndpoint" + + network_interface = (known after apply) + + private_dns_zone_configs = (known after apply) + + resource_group_name = (known after apply) + + subnet_id = (known after apply) + + + private_dns_zone_group { + + id = (known after apply) + + name = "BlobPrivateDnsZoneGroup" + + private_dns_zone_ids = (known after apply) + } + + + private_service_connection { + + is_manual_connection = false + + name = "BlobStoragePrivateEndpointConnection" + + private_connection_resource_id = (known after apply) + + private_ip_address = (known after apply) + + subresource_names = [ + + "blob", + ] + } + } + + # module.container_registry.azurerm_container_registry.acr will be created + + resource "azurerm_container_registry" "acr" { + + admin_enabled = true + + admin_password = (sensitive value) + + admin_username = (known after apply) + + encryption = (known after apply) + + export_policy_enabled = true + + id = (known after apply) + + location = "westus3" + + login_server = (known after apply) + + name = (known after apply) + + network_rule_bypass_option = "AzureServices" + + network_rule_set = (known after apply) + + public_network_access_enabled = true + + resource_group_name = (known after apply) + + sku = "Premium" + + trust_policy_enabled = false + + zone_redundancy_enabled = false + + + identity { + + identity_ids = (known after apply) + + principal_id = (known after apply) + + tenant_id = (known after apply) + + type = "UserAssigned" + } + } + + # module.container_registry.azurerm_monitor_diagnostic_setting.settings will be created + + resource "azurerm_monitor_diagnostic_setting" "settings" { + + id = (known after apply) + + log_analytics_destination_type = (known after apply) + + log_analytics_workspace_id = (known after apply) + + name = "ContainerDiagnosticsSettings" + + target_resource_id = (known after apply) + + + enabled_log { + + category = "ContainerRegistryLoginEvents" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "ContainerRegistryRepositoryEvents" + # (1 unchanged attribute hidden) + } + + + metric { + + category = "AllMetrics" + + enabled = true + } + } + + # module.container_registry.azurerm_user_assigned_identity.acr_identity will be created + + resource "azurerm_user_assigned_identity" "acr_identity" { + + client_id = (known after apply) + + id = (known after apply) + + location = "westus3" + + name = (known after apply) + + principal_id = (known after apply) + + resource_group_name = (known after apply) + + tenant_id = (known after apply) + } + + # module.key_vault.azurerm_key_vault.key_vault will be created + + resource "azurerm_key_vault" "key_vault" { + + access_policy = (known after apply) + + enable_rbac_authorization = true + + enabled_for_deployment = true + + enabled_for_disk_encryption = true + + enabled_for_template_deployment = true + + id = (known after apply) + + location = "westus3" + + name = (known after apply) + + public_network_access_enabled = true + + purge_protection_enabled = false + + resource_group_name = (known after apply) + + sku_name = "standard" + + soft_delete_retention_days = 30 + + tenant_id = "72f988bf-86f1-41af-91ab-2d7cd011db47" + + vault_uri = (known after apply) + + + contact (known after apply) + + + network_acls { + + bypass = "AzureServices" + + default_action = "Allow" + } + + + timeouts { + + delete = "60m" + } + } + + # module.key_vault.azurerm_monitor_diagnostic_setting.settings will be created + + resource "azurerm_monitor_diagnostic_setting" "settings" { + + id = (known after apply) + + log_analytics_destination_type = (known after apply) + + log_analytics_workspace_id = (known after apply) + + name = "KeyVaultDiagnosticsSettings" + + target_resource_id = (known after apply) + + + enabled_log { + + category = "AuditEvent" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "AzurePolicyEvaluationDetails" + # (1 unchanged attribute hidden) + } + + + metric { + + category = "AllMetrics" + + enabled = true + } + } + + # module.key_vault_private_dns_zone.azurerm_private_dns_zone.private_dns_zone will be created + + resource "azurerm_private_dns_zone" "private_dns_zone" { + + id = (known after apply) + + max_number_of_record_sets = (known after apply) + + max_number_of_virtual_network_links = (known after apply) + + max_number_of_virtual_network_links_with_registration = (known after apply) + + name = "privatelink.vaultcore.azure.net" + + number_of_record_sets = (known after apply) + + resource_group_name = (known after apply) + + + soa_record (known after apply) + } + + # module.key_vault_private_dns_zone.azurerm_private_dns_zone_virtual_network_link.link["AksVNet"] will be created + + resource "azurerm_private_dns_zone_virtual_network_link" "link" { + + id = (known after apply) + + name = "link_to_aksvnet" + + private_dns_zone_name = "privatelink.vaultcore.azure.net" + + registration_enabled = false + + resource_group_name = (known after apply) + + virtual_network_id = (known after apply) + } + + # module.key_vault_private_endpoint.azurerm_private_endpoint.private_endpoint will be created + + resource "azurerm_private_endpoint" "private_endpoint" { + + custom_dns_configs = (known after apply) + + id = (known after apply) + + location = "westus3" + + name = "VaultPrivateEndpoint" + + network_interface = (known after apply) + + private_dns_zone_configs = (known after apply) + + resource_group_name = (known after apply) + + subnet_id = (known after apply) + + + private_dns_zone_group { + + id = (known after apply) + + name = "KeyVaultPrivateDnsZoneGroup" + + private_dns_zone_ids = (known after apply) + } + + + private_service_connection { + + is_manual_connection = false + + name = "VaultPrivateEndpointConnection" + + private_connection_resource_id = (known after apply) + + private_ip_address = (known after apply) + + subresource_names = [ + + "vault", + ] + } + } + + # module.log_analytics_workspace.azurerm_log_analytics_solution.la_solution["ContainerInsights"] will be created + + resource "azurerm_log_analytics_solution" "la_solution" { + + id = (known after apply) + + location = "westus3" + + resource_group_name = (known after apply) + + solution_name = "ContainerInsights" + + workspace_name = "Workspace" + + workspace_resource_id = (known after apply) + + + plan { + + name = (known after apply) + + product = "OMSGallery/ContainerInsights" + + publisher = "Microsoft" + } + } + + # module.log_analytics_workspace.azurerm_log_analytics_workspace.log_analytics_workspace will be created + + resource "azurerm_log_analytics_workspace" "log_analytics_workspace" { + + allow_resource_only_permissions = true + + daily_quota_gb = -1 + + id = (known after apply) + + internet_ingestion_enabled = true + + internet_query_enabled = true + + local_authentication_disabled = false + + location = "westus3" + + name = "Workspace" + + primary_shared_key = (sensitive value) + + resource_group_name = (known after apply) + + retention_in_days = 30 + + secondary_shared_key = (sensitive value) + + sku = "PerGB2018" + + workspace_id = (known after apply) + } + + # module.nat_gateway.azurerm_nat_gateway.nat_gateway will be created + + resource "azurerm_nat_gateway" "nat_gateway" { + + id = (known after apply) + + idle_timeout_in_minutes = 4 + + location = "westus3" + + name = "NatGateway" + + resource_group_name = (known after apply) + + resource_guid = (known after apply) + + sku_name = "Standard" + + zones = [ + + "1", + ] + } + + # module.nat_gateway.azurerm_nat_gateway_public_ip_association.nat_gategay_public_ip_association will be created + + resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_association" { + + id = (known after apply) + + nat_gateway_id = (known after apply) + + public_ip_address_id = (known after apply) + } + + # module.nat_gateway.azurerm_public_ip.nat_gategay_public_ip will be created + + resource "azurerm_public_ip" "nat_gategay_public_ip" { + + allocation_method = "Static" + + ddos_protection_mode = "VirtualNetworkInherited" + + fqdn = (known after apply) + + id = (known after apply) + + idle_timeout_in_minutes = 4 + + ip_address = (known after apply) + + ip_version = "IPv4" + + location = "westus3" + + name = "NatGatewayPublicIp" + + resource_group_name = (known after apply) + + sku = "Standard" + + sku_tier = "Regional" + + zones = [ + + "1", + ] + } + + # module.nat_gateway.azurerm_subnet_nat_gateway_association.nat-avd-sessionhosts["AzureBastionSubnet"] will be created + + resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { + + id = (known after apply) + + nat_gateway_id = (known after apply) + + subnet_id = (known after apply) + } + + # module.nat_gateway.azurerm_subnet_nat_gateway_association.nat-avd-sessionhosts["PodSubnet"] will be created + + resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { + + id = (known after apply) + + nat_gateway_id = (known after apply) + + subnet_id = (known after apply) + } + + # module.nat_gateway.azurerm_subnet_nat_gateway_association.nat-avd-sessionhosts["SystemSubnet"] will be created + + resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { + + id = (known after apply) + + nat_gateway_id = (known after apply) + + subnet_id = (known after apply) + } + + # module.nat_gateway.azurerm_subnet_nat_gateway_association.nat-avd-sessionhosts["UserSubnet"] will be created + + resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { + + id = (known after apply) + + nat_gateway_id = (known after apply) + + subnet_id = (known after apply) + } + + # module.nat_gateway.azurerm_subnet_nat_gateway_association.nat-avd-sessionhosts["VmSubnet"] will be created + + resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { + + id = (known after apply) + + nat_gateway_id = (known after apply) + + subnet_id = (known after apply) + } + + # module.openai.azurerm_cognitive_account.openai will be created + + resource "azurerm_cognitive_account" "openai" { + + custom_subdomain_name = "magic8ball" + + endpoint = (known after apply) + + id = (known after apply) + + kind = "OpenAI" + + local_auth_enabled = true + + location = "westus3" + + name = (known after apply) + + outbound_network_access_restricted = false + + primary_access_key = (sensitive value) + + public_network_access_enabled = true + + resource_group_name = (known after apply) + + secondary_access_key = (sensitive value) + + sku_name = "S0" + + + identity { + + principal_id = (known after apply) + + tenant_id = (known after apply) + + type = "SystemAssigned" + } + } + + # module.openai.azurerm_cognitive_deployment.deployment["gpt-4"] will be created + + resource "azurerm_cognitive_deployment" "deployment" { + + cognitive_account_id = (known after apply) + + id = (known after apply) + + name = "gpt-4" + + version_upgrade_option = "OnceNewDefaultVersionAvailable" + + + model { + + format = "OpenAI" + + name = "gpt-4" + + version = "turbo-2024-04-09" + } + + + sku { + + capacity = 1 + + name = "Standard" + } + } + + # module.openai.azurerm_monitor_diagnostic_setting.settings will be created + + resource "azurerm_monitor_diagnostic_setting" "settings" { + + id = (known after apply) + + log_analytics_destination_type = (known after apply) + + log_analytics_workspace_id = (known after apply) + + name = "OpenAiDiagnosticsSettings" + + target_resource_id = (known after apply) + + + enabled_log { + + category = "Audit" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "RequestResponse" + # (1 unchanged attribute hidden) + } + + enabled_log { + + category = "Trace" + # (1 unchanged attribute hidden) + } + + + metric { + + category = "AllMetrics" + + enabled = true + } + } + + # module.openai_private_dns_zone.azurerm_private_dns_zone.private_dns_zone will be created + + resource "azurerm_private_dns_zone" "private_dns_zone" { + + id = (known after apply) + + max_number_of_record_sets = (known after apply) + + max_number_of_virtual_network_links = (known after apply) + + max_number_of_virtual_network_links_with_registration = (known after apply) + + name = "privatelink.openai.azure.com" + + number_of_record_sets = (known after apply) + + resource_group_name = (known after apply) + + + soa_record (known after apply) + } + + # module.openai_private_dns_zone.azurerm_private_dns_zone_virtual_network_link.link["AksVNet"] will be created + + resource "azurerm_private_dns_zone_virtual_network_link" "link" { + + id = (known after apply) + + name = "link_to_aksvnet" + + private_dns_zone_name = "privatelink.openai.azure.com" + + registration_enabled = false + + resource_group_name = (known after apply) + + virtual_network_id = (known after apply) + } + + # module.openai_private_endpoint.azurerm_private_endpoint.private_endpoint will be created + + resource "azurerm_private_endpoint" "private_endpoint" { + + custom_dns_configs = (known after apply) + + id = (known after apply) + + location = "westus3" + + name = "OpenAiPrivateEndpoint" + + network_interface = (known after apply) + + private_dns_zone_configs = (known after apply) + + resource_group_name = (known after apply) + + subnet_id = (known after apply) + + + private_dns_zone_group { + + id = (known after apply) + + name = "AcrPrivateDnsZoneGroup" + + private_dns_zone_ids = (known after apply) + } + + + private_service_connection { + + is_manual_connection = false + + name = "OpenAiPrivateEndpointConnection" + + private_connection_resource_id = (known after apply) + + private_ip_address = (known after apply) + + subresource_names = [ + + "account", + ] + } + } + + # module.storage_account.azurerm_storage_account.storage_account will be created + + resource "azurerm_storage_account" "storage_account" { + + access_tier = (known after apply) + + account_kind = "StorageV2" + + account_replication_type = "LRS" + + account_tier = "Standard" + + allow_nested_items_to_be_public = false + + cross_tenant_replication_enabled = false + + default_to_oauth_authentication = false + + dns_endpoint_type = "Standard" + + https_traffic_only_enabled = true + + id = (known after apply) + + infrastructure_encryption_enabled = false + + is_hns_enabled = false + + large_file_share_enabled = (known after apply) + + local_user_enabled = true + + location = "westus3" + + min_tls_version = "TLS1_2" + + name = (known after apply) + + nfsv3_enabled = false + + primary_access_key = (sensitive value) + + primary_blob_connection_string = (sensitive value) + + primary_blob_endpoint = (known after apply) + + primary_blob_host = (known after apply) + + primary_blob_internet_endpoint = (known after apply) + + primary_blob_internet_host = (known after apply) + + primary_blob_microsoft_endpoint = (known after apply) + + primary_blob_microsoft_host = (known after apply) + + primary_connection_string = (sensitive value) + + primary_dfs_endpoint = (known after apply) + + primary_dfs_host = (known after apply) + + primary_dfs_internet_endpoint = (known after apply) + + primary_dfs_internet_host = (known after apply) + + primary_dfs_microsoft_endpoint = (known after apply) + + primary_dfs_microsoft_host = (known after apply) + + primary_file_endpoint = (known after apply) + + primary_file_host = (known after apply) + + primary_file_internet_endpoint = (known after apply) + + primary_file_internet_host = (known after apply) + + primary_file_microsoft_endpoint = (known after apply) + + primary_file_microsoft_host = (known after apply) + + primary_location = (known after apply) + + primary_queue_endpoint = (known after apply) + + primary_queue_host = (known after apply) + + primary_queue_microsoft_endpoint = (known after apply) + + primary_queue_microsoft_host = (known after apply) + + primary_table_endpoint = (known after apply) + + primary_table_host = (known after apply) + + primary_table_microsoft_endpoint = (known after apply) + + primary_table_microsoft_host = (known after apply) + + primary_web_endpoint = (known after apply) + + primary_web_host = (known after apply) + + primary_web_internet_endpoint = (known after apply) + + primary_web_internet_host = (known after apply) + + primary_web_microsoft_endpoint = (known after apply) + + primary_web_microsoft_host = (known after apply) + + public_network_access_enabled = true + + queue_encryption_key_type = "Service" + + resource_group_name = (known after apply) + + secondary_access_key = (sensitive value) + + secondary_blob_connection_string = (sensitive value) + + secondary_blob_endpoint = (known after apply) + + secondary_blob_host = (known after apply) + + secondary_blob_internet_endpoint = (known after apply) + + secondary_blob_internet_host = (known after apply) + + secondary_blob_microsoft_endpoint = (known after apply) + + secondary_blob_microsoft_host = (known after apply) + + secondary_connection_string = (sensitive value) + + secondary_dfs_endpoint = (known after apply) + + secondary_dfs_host = (known after apply) + + secondary_dfs_internet_endpoint = (known after apply) + + secondary_dfs_internet_host = (known after apply) + + secondary_dfs_microsoft_endpoint = (known after apply) + + secondary_dfs_microsoft_host = (known after apply) + + secondary_file_endpoint = (known after apply) + + secondary_file_host = (known after apply) + + secondary_file_internet_endpoint = (known after apply) + + secondary_file_internet_host = (known after apply) + + secondary_file_microsoft_endpoint = (known after apply) + + secondary_file_microsoft_host = (known after apply) + + secondary_location = (known after apply) + + secondary_queue_endpoint = (known after apply) + + secondary_queue_host = (known after apply) + + secondary_queue_microsoft_endpoint = (known after apply) + + secondary_queue_microsoft_host = (known after apply) + + secondary_table_endpoint = (known after apply) + + secondary_table_host = (known after apply) + + secondary_table_microsoft_endpoint = (known after apply) + + secondary_table_microsoft_host = (known after apply) + + secondary_web_endpoint = (known after apply) + + secondary_web_host = (known after apply) + + secondary_web_internet_endpoint = (known after apply) + + secondary_web_internet_host = (known after apply) + + secondary_web_microsoft_endpoint = (known after apply) + + secondary_web_microsoft_host = (known after apply) + + sftp_enabled = false + + shared_access_key_enabled = true + + table_encryption_key_type = "Service" + + + blob_properties (known after apply) + + + identity { + + principal_id = (known after apply) + + tenant_id = (known after apply) + + type = "SystemAssigned" + } + + + network_rules (known after apply) + + + queue_properties (known after apply) + + + routing (known after apply) + + + share_properties (known after apply) + + + static_website (known after apply) + } + + # module.virtual_network.azurerm_monitor_diagnostic_setting.settings will be created + + resource "azurerm_monitor_diagnostic_setting" "settings" { + + id = (known after apply) + + log_analytics_destination_type = (known after apply) + + log_analytics_workspace_id = (known after apply) + + name = "VirtualNetworkDiagnosticsSettings" + + target_resource_id = (known after apply) + + + metric { + + category = "AllMetrics" + + enabled = true + } + } + + # module.virtual_network.azurerm_subnet.subnet["AzureBastionSubnet"] will be created + + resource "azurerm_subnet" "subnet" { + + address_prefixes = [ + + "10.243.2.0/24", + ] + + default_outbound_access_enabled = true + + id = (known after apply) + + name = "AzureBastionSubnet" + + private_endpoint_network_policies = "Enabled" + + private_link_service_network_policies_enabled = false + + resource_group_name = (known after apply) + + virtual_network_name = "AksVNet" + } + + # module.virtual_network.azurerm_subnet.subnet["PodSubnet"] will be created + + resource "azurerm_subnet" "subnet" { + + address_prefixes = [ + + "10.242.0.0/16", + ] + + default_outbound_access_enabled = true + + id = (known after apply) + + name = "PodSubnet" + + private_endpoint_network_policies = "Enabled" + + private_link_service_network_policies_enabled = false + + resource_group_name = (known after apply) + + virtual_network_name = "AksVNet" + + + delegation { + + name = "delegation" + + + service_delegation { + + actions = [ + + "Microsoft.Network/virtualNetworks/subnets/join/action", + ] + + name = "Microsoft.ContainerService/managedClusters" + } + } + } + + # module.virtual_network.azurerm_subnet.subnet["SystemSubnet"] will be created + + resource "azurerm_subnet" "subnet" { + + address_prefixes = [ + + "10.240.0.0/16", + ] + + default_outbound_access_enabled = true + + id = (known after apply) + + name = "SystemSubnet" + + private_endpoint_network_policies = "Enabled" + + private_link_service_network_policies_enabled = false + + resource_group_name = (known after apply) + + virtual_network_name = "AksVNet" + } + + # module.virtual_network.azurerm_subnet.subnet["UserSubnet"] will be created + + resource "azurerm_subnet" "subnet" { + + address_prefixes = [ + + "10.241.0.0/16", + ] + + default_outbound_access_enabled = true + + id = (known after apply) + + name = "UserSubnet" + + private_endpoint_network_policies = "Enabled" + + private_link_service_network_policies_enabled = false + + resource_group_name = (known after apply) + + virtual_network_name = "AksVNet" + } + + # module.virtual_network.azurerm_subnet.subnet["VmSubnet"] will be created + + resource "azurerm_subnet" "subnet" { + + address_prefixes = [ + + "10.243.1.0/24", + ] + + default_outbound_access_enabled = true + + id = (known after apply) + + name = "VmSubnet" + + private_endpoint_network_policies = "Enabled" + + private_link_service_network_policies_enabled = false + + resource_group_name = (known after apply) + + virtual_network_name = "AksVNet" + } + + # module.virtual_network.azurerm_virtual_network.vnet will be created + + resource "azurerm_virtual_network" "vnet" { + + address_space = [ + + "10.0.0.0/8", + ] + + dns_servers = (known after apply) + + guid = (known after apply) + + id = (known after apply) + + location = "westus3" + + name = "AksVNet" + + private_endpoint_vnet_policies = "Disabled" + + resource_group_name = (known after apply) + + subnet = (known after apply) + } \ No newline at end of file From b526002c6e98edc3f8904094a04ab30135a2d0a1 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 22 Jan 2025 23:06:30 -0500 Subject: [PATCH 051/119] Clean --- scenarios/AksOpenAiTerraform/terraform/main.tf | 11 +++++------ .../terraform/modules/diagnostic_setting/main.tf | 16 ---------------- .../terraform/modules/private_endpoint/main.tf | 1 + 3 files changed, 6 insertions(+), 22 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index c62f844a1..79a417266 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -100,11 +100,6 @@ module "aks_cluster" { pod_subnet_id = module.virtual_network.subnet_ids[local.pod_subnet_name] log_analytics_workspace_id = module.log_analytics_workspace.id - - depends_on = [ - module.nat_gateway, - module.container_registry - ] } module "container_registry" { @@ -294,10 +289,11 @@ module "openai_private_endpoint" { name = "OpenAiPrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.openai.id subresource_name = "account" - private_dns_zone_group_name = "AcrPrivateDnsZoneGroup" + private_dns_zone_group_name = "OpenAiPrivateDnsZoneGroup" private_dns_zone_group_ids = [module.openai_private_dns_zone.id] } @@ -306,6 +302,7 @@ module "acr_private_endpoint" { name = "AcrPrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.container_registry.id subresource_name = "registry" @@ -318,6 +315,7 @@ module "key_vault_private_endpoint" { name = "VaultPrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.key_vault.id subresource_name = "vault" @@ -330,6 +328,7 @@ module "blob_private_endpoint" { name = "BlobStoragePrivateEndpoint" location = var.location resource_group_name = azurerm_resource_group.rg.name + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] private_connection_resource_id = module.storage_account.id subresource_name = "blob" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf index 3f8f5af32..c188cf7ac 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf @@ -9,20 +9,4 @@ resource "azurerm_monitor_diagnostic_setting" "settings" { eventhub_authorization_rule_id = var.eventhub_authorization_rule_id storage_account_id = var.storage_account_id - - dynamic "log" { - for_each = toset(logs) - content { - category = each.key - enabled = true - } - } - - dynamic "metric" { - for_each = toset(metrics) - content { - category = each.key - enabled = true - } - } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf index c73bdaefd..44f311a46 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf @@ -2,6 +2,7 @@ resource "azurerm_private_endpoint" "private_endpoint" { name = var.name location = var.location resource_group_name = var.resource_group_name + subnet_id = var.subnet_id private_service_connection { From faffb2b5b8ebff3135f0b43748bd9945d95725e8 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 00:31:05 -0500 Subject: [PATCH 052/119] Clean up --- .../AksOpenAiTerraform/terraform/main.tf | 121 ++++++------------ .../terraform/modules/dns_zone/main.tf | 30 +++++ .../variables.tf | 16 +-- .../modules/private_dns_zone/main.tf | 13 -- .../modules/private_dns_zone/outputs.tf | 4 - .../modules/private_dns_zone/variables.tf | 14 -- .../modules/private_endpoint/main.tf | 19 --- .../modules/virtual_network/outputs.tf | 4 + 8 files changed, 80 insertions(+), 141 deletions(-) create mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/main.tf rename scenarios/AksOpenAiTerraform/terraform/modules/{private_endpoint => dns_zone}/variables.tf (84%) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 79a417266..e87cbcc7b 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -100,6 +100,8 @@ module "aks_cluster" { pod_subnet_id = module.virtual_network.subnet_ids[local.pod_subnet_name] log_analytics_workspace_id = module.log_analytics_workspace.id + + depends_on = [module.nat_gateway] } module "container_registry" { @@ -234,106 +236,59 @@ module "bastion_host" { # Private DNS Zones ############################################################################### module "acr_private_dns_zone" { - source = "./modules/private_dns_zone" - name = "privatelink.azurecr.io" + source = "./modules/dns_zone" + location = var.location resource_group_name = azurerm_resource_group.rg.name - virtual_networks_to_link = { - (module.virtual_network.name) = { - subscription_id = local.subscription_id - resource_group_name = azurerm_resource_group.rg.name - } - } -} -module "openai_private_dns_zone" { - source = "./modules/private_dns_zone" - name = "privatelink.openai.azure.com" - resource_group_name = azurerm_resource_group.rg.name - virtual_networks_to_link = { - (module.virtual_network.name) = { - subscription_id = local.subscription_id - resource_group_name = azurerm_resource_group.rg.name - } - } -} + name = "privatelink.azurecr.io" + private_dns_zone_group_name = "OpenAiPrivateDnsZoneGroup" + subresource_name = "account" + private_connection_resource_id = module.openai.id -module "key_vault_private_dns_zone" { - source = "./modules/private_dns_zone" - name = "privatelink.vaultcore.azure.net" - resource_group_name = azurerm_resource_group.rg.name - virtual_networks_to_link = { - (module.virtual_network.name) = { - subscription_id = local.subscription_id - resource_group_name = azurerm_resource_group.rg.name - } - } + virtual_network_id = module.virtual_network.id + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] } -module "blob_private_dns_zone" { - source = "./modules/private_dns_zone" - name = "privatelink.blob.core.windows.net" +module "openai_private_dns_zone" { + source = "./modules/dns_zone" + location = var.location resource_group_name = azurerm_resource_group.rg.name - virtual_networks_to_link = { - (module.virtual_network.name) = { - subscription_id = local.subscription_id - resource_group_name = azurerm_resource_group.rg.name - } - } -} - -############################################################################### -# Private Endpoints -############################################################################### -module "openai_private_endpoint" { - source = "./modules/private_endpoint" - name = "OpenAiPrivateEndpoint" - location = var.location - resource_group_name = azurerm_resource_group.rg.name + name = "privatelink.openai.azure.com" + private_dns_zone_group_name = "AcrPrivateDnsZoneGroup" + subresource_name = "registry" + private_connection_resource_id = module.container_registry.id + + virtual_network_id = module.virtual_network.id subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] - private_connection_resource_id = module.openai.id - subresource_name = "account" - private_dns_zone_group_name = "OpenAiPrivateDnsZoneGroup" - private_dns_zone_group_ids = [module.openai_private_dns_zone.id] } -module "acr_private_endpoint" { - source = "./modules/private_endpoint" - name = "AcrPrivateEndpoint" - location = var.location - resource_group_name = azurerm_resource_group.rg.name - - subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] - private_connection_resource_id = module.container_registry.id - subresource_name = "registry" - private_dns_zone_group_name = "AcrPrivateDnsZoneGroup" - private_dns_zone_group_ids = [module.acr_private_dns_zone.id] -} +module "key_vault_private_dns_zone" { + source = "./modules/dns_zone" + location = var.location + resource_group_name = azurerm_resource_group.rg.name -module "key_vault_private_endpoint" { - source = "./modules/private_endpoint" - name = "VaultPrivateEndpoint" - location = var.location - resource_group_name = azurerm_resource_group.rg.name + name = "privatelink.vaultcore.azure.net" + private_dns_zone_group_name = "KeyVaultPrivateDnsZoneGroup" + subresource_name = "vault" + private_connection_resource_id = module.key_vault.id + virtual_network_id = module.virtual_network.id subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] - private_connection_resource_id = module.key_vault.id - subresource_name = "vault" - private_dns_zone_group_name = "KeyVaultPrivateDnsZoneGroup" - private_dns_zone_group_ids = [module.key_vault_private_dns_zone.id] } -module "blob_private_endpoint" { - source = "./modules/private_endpoint" - name = "BlobStoragePrivateEndpoint" - location = var.location - resource_group_name = azurerm_resource_group.rg.name +module "blob_private_dns_zone" { + source = "./modules/dns_zone" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + + name = "privatelink.blob.core.windows.net" + private_dns_zone_group_name = "BlobPrivateDnsZoneGroup" + subresource_name = "blob" + private_connection_resource_id = module.storage_account.id + virtual_network_id = module.virtual_network.id subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] - private_connection_resource_id = module.storage_account.id - subresource_name = "blob" - private_dns_zone_group_name = "BlobPrivateDnsZoneGroup" - private_dns_zone_group_ids = [module.blob_private_dns_zone.id] } ############################################################################### diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/main.tf new file mode 100644 index 000000000..7b205c3c2 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/main.tf @@ -0,0 +1,30 @@ +resource "azurerm_private_dns_zone" "this" { + name = var.name + resource_group_name = var.resource_group_name +} + +resource "azurerm_private_dns_zone_virtual_network_link" "this" { + name = "link_to_${lower(basename(azurerm_private_dns_zone.this.name))}" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.this.name + virtual_network_id = var.virtual_network_id +} + +resource "azurerm_private_endpoint" "this" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.subnet_id + + private_service_connection { + name = "${var.name}Connection" + private_connection_resource_id = var.private_connection_resource_id + is_manual_connection = false + subresource_names = [var.subresource_name] + } + + private_dns_zone_group { + name = azurerm_private_dns_zone.this.name + private_dns_zone_ids = [var.private_connection_resource_id] + } +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/variables.tf similarity index 84% rename from scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf rename to scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/variables.tf index 8bc78cbef..72a927f80 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/variables.tf @@ -2,30 +2,30 @@ variable "name" { type = string } -variable "resource_group_name" { +variable "location" { type = string } -variable "private_connection_resource_id" { +variable "resource_group_name" { type = string } -variable "location" { +variable "private_dns_zone_group_name" { type = string } -variable "subnet_id" { +variable "subresource_name" { type = string } -variable "subresource_name" { +variable "virtual_network_id" { type = string } -variable "private_dns_zone_group_name" { +variable "subnet_id" { type = string } -variable "private_dns_zone_group_ids" { - type = list(string) +variable "private_connection_resource_id" { + type = string } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf deleted file mode 100644 index be1d6a7ea..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/main.tf +++ /dev/null @@ -1,13 +0,0 @@ -resource "azurerm_private_dns_zone" "private_dns_zone" { - name = var.name - resource_group_name = var.resource_group_name -} - -resource "azurerm_private_dns_zone_virtual_network_link" "link" { - for_each = var.virtual_networks_to_link - - name = "link_to_${lower(basename(each.key))}" - resource_group_name = var.resource_group_name - private_dns_zone_name = azurerm_private_dns_zone.private_dns_zone.name - virtual_network_id = "/subscriptions/${each.value.subscription_id}/resourceGroups/${each.value.resource_group_name}/providers/Microsoft.Network/virtualNetworks/${each.key}" -} diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/outputs.tf deleted file mode 100644 index c37a77f92..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "id" { - description = "Specifies the resource id of the private dns zone" - value = azurerm_private_dns_zone.private_dns_zone.id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf deleted file mode 100644 index ce748b6f9..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_dns_zone/variables.tf +++ /dev/null @@ -1,14 +0,0 @@ -variable "name" { - type = string -} - -variable "resource_group_name" { - type = string -} - -variable "virtual_networks_to_link" { - type = map(object({ - subscription_id = string - resource_group_name = string - })) -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf deleted file mode 100644 index 44f311a46..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/private_endpoint/main.tf +++ /dev/null @@ -1,19 +0,0 @@ -resource "azurerm_private_endpoint" "private_endpoint" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - - subnet_id = var.subnet_id - - private_service_connection { - name = "${var.name}Connection" - private_connection_resource_id = var.private_connection_resource_id - is_manual_connection = false - subresource_names = [var.subresource_name] - } - - private_dns_zone_group { - name = var.private_dns_zone_group_name - private_dns_zone_ids = var.private_dns_zone_group_ids - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf index b8d3adc64..32c5d99f0 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf @@ -2,6 +2,10 @@ output "name" { value = azurerm_virtual_network.vnet.name } +output "id" { + value = azurerm_virtual_network.vnet.id +} + output "subnet_ids" { value = { for subnet in azurerm_subnet.subnet : subnet.name => subnet.id } } \ No newline at end of file From f042cf6636a4dff103d5bb622a4966a8f8bd5977 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 00:32:31 -0500 Subject: [PATCH 053/119] Fix --- scenarios/AksOpenAiTerraform/terraform/main.tf | 2 +- scenarios/AksOpenAiTerraform/terraform/variables.tf | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index e87cbcc7b..8abb21b5b 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -75,7 +75,7 @@ module "openai" { } } ] - custom_subdomain_name = "magic8ball" + custom_subdomain_name = var.openai_subdomain public_network_access_enabled = true log_analytics_workspace_id = module.log_analytics_workspace.id diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index af24bc583..55085a5e0 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -8,6 +8,11 @@ variable "location" { default = "westus3" } +variable "openai_subdomain" { + type = string + default = "" +} + variable "kubernetes_version" { type = string default = "1.30.7" From 6f69f62eab7061294fe32775180bdc9dd05b90fd Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 00:54:31 -0500 Subject: [PATCH 054/119] Fix bug --- scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/main.tf index 7b205c3c2..6174d22a5 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/main.tf @@ -25,6 +25,6 @@ resource "azurerm_private_endpoint" "this" { private_dns_zone_group { name = azurerm_private_dns_zone.this.name - private_dns_zone_ids = [var.private_connection_resource_id] + private_dns_zone_ids = [azurerm_private_dns_zone.this.id] } } \ No newline at end of file From aec0897ac13ba35de27975cf02a7b50b29d72b4b Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 01:10:42 -0500 Subject: [PATCH 055/119] Move --- scenarios/AksOpenAiTerraform/terraform/main.tf | 8 ++++---- .../terraform/modules/{dns_zone => dns}/main.tf | 0 .../terraform/modules/{dns_zone => dns}/variables.tf | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename scenarios/AksOpenAiTerraform/terraform/modules/{dns_zone => dns}/main.tf (100%) rename scenarios/AksOpenAiTerraform/terraform/modules/{dns_zone => dns}/variables.tf (100%) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 8abb21b5b..d61d99803 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -236,7 +236,7 @@ module "bastion_host" { # Private DNS Zones ############################################################################### module "acr_private_dns_zone" { - source = "./modules/dns_zone" + source = "./modules/dns" location = var.location resource_group_name = azurerm_resource_group.rg.name @@ -250,7 +250,7 @@ module "acr_private_dns_zone" { } module "openai_private_dns_zone" { - source = "./modules/dns_zone" + source = "./modules/dns" location = var.location resource_group_name = azurerm_resource_group.rg.name @@ -264,7 +264,7 @@ module "openai_private_dns_zone" { } module "key_vault_private_dns_zone" { - source = "./modules/dns_zone" + source = "./modules/dns" location = var.location resource_group_name = azurerm_resource_group.rg.name @@ -278,7 +278,7 @@ module "key_vault_private_dns_zone" { } module "blob_private_dns_zone" { - source = "./modules/dns_zone" + source = "./modules/dns" location = var.location resource_group_name = azurerm_resource_group.rg.name diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf similarity index 100% rename from scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/main.tf rename to scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf similarity index 100% rename from scenarios/AksOpenAiTerraform/terraform/modules/dns_zone/variables.tf rename to scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf From 2f9c8eb7dcc731f5f603d159fbb2dab5ede65700 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 01:13:53 -0500 Subject: [PATCH 056/119] Fix bug --- .../AksOpenAiTerraform/terraform/modules/virtual_network/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf index af0cdc680..ef02041d7 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf @@ -13,7 +13,6 @@ resource "azurerm_subnet" "subnet" { virtual_network_name = azurerm_virtual_network.vnet.name address_prefixes = each.value.address_prefixes private_endpoint_network_policies = "Enabled" - private_link_service_network_policies_enabled = false dynamic "delegation" { for_each = each.value.delegation != null ? [each.value.delegation] : [] From 8ef4d065d1a7ede7ef411a172f182bceba141bf3 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 01:32:33 -0500 Subject: [PATCH 057/119] Fixes --- .../AksOpenAiTerraform/terraform/main.tf | 34 +++++++++---------- .../terraform/modules/dns/main.tf | 4 +-- .../terraform/modules/dns/variables.tf | 2 +- .../terraform/modules/virtual_network/main.tf | 10 +++--- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index d61d99803..7d99ba1a1 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -237,58 +237,58 @@ module "bastion_host" { ############################################################################### module "acr_private_dns_zone" { source = "./modules/dns" + name = "OpenAiPrivateDnsZone" location = var.location resource_group_name = azurerm_resource_group.rg.name - name = "privatelink.azurecr.io" - private_dns_zone_group_name = "OpenAiPrivateDnsZoneGroup" + endpoint = "privatelink.azurecr.io" subresource_name = "account" private_connection_resource_id = module.openai.id - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] + virtual_network_id = module.virtual_network.id + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] } module "openai_private_dns_zone" { source = "./modules/dns" + name = "AcrPrivateDnsZone" location = var.location resource_group_name = azurerm_resource_group.rg.name - name = "privatelink.openai.azure.com" - private_dns_zone_group_name = "AcrPrivateDnsZoneGroup" + endpoint = "privatelink.openai.azure.com" subresource_name = "registry" private_connection_resource_id = module.container_registry.id - - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] + + virtual_network_id = module.virtual_network.id + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] } module "key_vault_private_dns_zone" { source = "./modules/dns" + name = "KeyVaultPrivateDnsZone" location = var.location resource_group_name = azurerm_resource_group.rg.name - name = "privatelink.vaultcore.azure.net" - private_dns_zone_group_name = "KeyVaultPrivateDnsZoneGroup" + endpoint = "privatelink.vaultcore.azure.net" subresource_name = "vault" private_connection_resource_id = module.key_vault.id - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] + virtual_network_id = module.virtual_network.id + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] } module "blob_private_dns_zone" { source = "./modules/dns" + name = "BlobPrivateDnsZone" location = var.location resource_group_name = azurerm_resource_group.rg.name - name = "privatelink.blob.core.windows.net" - private_dns_zone_group_name = "BlobPrivateDnsZoneGroup" + endpoint = "privatelink.blob.core.windows.net" subresource_name = "blob" private_connection_resource_id = module.storage_account.id - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] + virtual_network_id = module.virtual_network.id + subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] } ############################################################################### diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf index 6174d22a5..5e0a3a3ee 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf @@ -11,7 +11,7 @@ resource "azurerm_private_dns_zone_virtual_network_link" "this" { } resource "azurerm_private_endpoint" "this" { - name = var.name + name = var.endpoint location = var.location resource_group_name = var.resource_group_name subnet_id = var.subnet_id @@ -24,7 +24,7 @@ resource "azurerm_private_endpoint" "this" { } private_dns_zone_group { - name = azurerm_private_dns_zone.this.name + name = "${var.name}Group" private_dns_zone_ids = [azurerm_private_dns_zone.this.id] } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf index 72a927f80..42557e820 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf @@ -10,7 +10,7 @@ variable "resource_group_name" { type = string } -variable "private_dns_zone_group_name" { +variable "endpoint" { type = string } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf index ef02041d7..30f5fe5cd 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf @@ -8,11 +8,11 @@ resource "azurerm_virtual_network" "vnet" { resource "azurerm_subnet" "subnet" { for_each = { for subnet in var.subnets : subnet.name => subnet } - name = each.key - resource_group_name = var.resource_group_name - virtual_network_name = azurerm_virtual_network.vnet.name - address_prefixes = each.value.address_prefixes - private_endpoint_network_policies = "Enabled" + name = each.key + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = each.value.address_prefixes + private_endpoint_network_policies = "Enabled" dynamic "delegation" { for_each = each.value.delegation != null ? [each.value.delegation] : [] From 1dca7d974f52d9ee3ef56dfc4c4af68ab24902d9 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 05:59:48 -0500 Subject: [PATCH 058/119] Fixes --- .../AksOpenAiTerraform/terraform/main.tf | 62 ++++++++----------- .../terraform/modules/aks/main.tf | 4 ++ .../terraform/modules/dns/main.tf | 9 +-- .../terraform/modules/dns/variables.tf | 4 -- .../AksOpenAiTerraform/terraform/variables.tf | 4 +- 5 files changed, 33 insertions(+), 50 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 7d99ba1a1..c226a0923 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -35,11 +35,6 @@ locals { subscription_id = data.azurerm_client_config.current.subscription_id random_id = random_string.rg_suffix.result - vm_subnet_name = "VmSubnet" - system_node_pool_subnet_name = "SystemSubnet" - user_node_pool_subnet_name = "UserSubnet" - pod_subnet_name = "PodSubnet" - namespace = "magic8ball" service_account_name = "magic8ball-sa" @@ -95,9 +90,9 @@ module "aks_cluster" { system_node_pool_vm_size = var.system_node_pool_vm_size user_node_pool_vm_size = var.user_node_pool_vm_size - system_node_pool_subnet_id = module.virtual_network.subnet_ids[local.system_node_pool_subnet_name] - user_node_pool_subnet_id = module.virtual_network.subnet_ids[local.user_node_pool_subnet_name] - pod_subnet_id = module.virtual_network.subnet_ids[local.pod_subnet_name] + system_node_pool_subnet_id = module.virtual_network.subnet_ids["SystemSubnet"] + user_node_pool_subnet_id = module.virtual_network.subnet_ids["UserSubnet"] + pod_subnet_id = module.virtual_network.subnet_ids["PodSubnet"] log_analytics_workspace_id = module.log_analytics_workspace.id @@ -173,32 +168,26 @@ module "virtual_network" { location = var.location resource_group_name = azurerm_resource_group.rg.name - log_analytics_workspace_id = module.log_analytics_workspace.id - address_space = ["10.0.0.0/8"] subnets = [ { - name : local.system_node_pool_subnet_name - address_prefixes : ["10.240.0.0/16"] - delegation = null + name : "VmSubnet" + address_prefixes : ["10.243.1.0/24"] }, { - name : local.user_node_pool_subnet_name - address_prefixes : ["10.241.0.0/16"] - delegation = null + name : "AzureBastionSubnet" + address_prefixes : ["10.243.2.0/24"] }, { - name : local.vm_subnet_name - address_prefixes : ["10.243.1.0/24"] - delegation = null + name : "SystemSubnet" + address_prefixes : ["10.240.0.0/16"] }, { - name : "AzureBastionSubnet" - address_prefixes : ["10.243.2.0/24"] - delegation = null + name : "UserSubnet" + address_prefixes : ["10.241.0.0/16"] }, { - name : local.pod_subnet_name + name : "PodSubnet" address_prefixes : ["10.242.0.0/16"] delegation = { name = "delegation" @@ -209,6 +198,8 @@ module "virtual_network" { } }, ] + + log_analytics_workspace_id = module.log_analytics_workspace.id } module "nat_gateway" { @@ -237,58 +228,54 @@ module "bastion_host" { ############################################################################### module "acr_private_dns_zone" { source = "./modules/dns" - name = "OpenAiPrivateDnsZone" location = var.location resource_group_name = azurerm_resource_group.rg.name - endpoint = "privatelink.azurecr.io" + name = "privatelink.azurecr.io" subresource_name = "account" private_connection_resource_id = module.openai.id virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] + subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } module "openai_private_dns_zone" { source = "./modules/dns" - name = "AcrPrivateDnsZone" location = var.location resource_group_name = azurerm_resource_group.rg.name - endpoint = "privatelink.openai.azure.com" + name = "privatelink.openai.azure.com" subresource_name = "registry" private_connection_resource_id = module.container_registry.id virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] + subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } module "key_vault_private_dns_zone" { source = "./modules/dns" - name = "KeyVaultPrivateDnsZone" location = var.location resource_group_name = azurerm_resource_group.rg.name - endpoint = "privatelink.vaultcore.azure.net" + name = "privatelink.vaultcore.azure.net" subresource_name = "vault" private_connection_resource_id = module.key_vault.id virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] + subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } module "blob_private_dns_zone" { source = "./modules/dns" - name = "BlobPrivateDnsZone" location = var.location resource_group_name = azurerm_resource_group.rg.name - endpoint = "privatelink.blob.core.windows.net" + name = "privatelink.blob.core.windows.net" subresource_name = "blob" private_connection_resource_id = module.storage_account.id virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids[local.vm_subnet_name] + subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } ############################################################################### @@ -303,6 +290,7 @@ resource "azurerm_user_assigned_identity" "aks_workload_identity" { resource "azurerm_federated_identity_credential" "federated_identity_credential" { name = "${title(local.namespace)}FederatedIdentity" resource_group_name = azurerm_resource_group.rg.name + audience = ["api://AzureADTokenExchange"] issuer = module.aks_cluster.oidc_issuer_url parent_id = azurerm_user_assigned_identity.aks_workload_identity.id @@ -310,14 +298,14 @@ resource "azurerm_federated_identity_credential" "federated_identity_credential" } resource "azurerm_role_assignment" "cognitive_services_user_assignment" { - scope = module.openai.id role_definition_name = "Cognitive Services User" + scope = module.openai.id principal_id = azurerm_user_assigned_identity.aks_workload_identity.principal_id } resource "azurerm_role_assignment" "network_contributor_assignment" { - scope = azurerm_resource_group.rg.id role_definition_name = "Network Contributor" + scope = azurerm_resource_group.rg.id principal_id = module.aks_cluster.aks_identity_principal_id } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index de4c20227..fdcd693e0 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -64,6 +64,10 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { keda_enabled = true vertical_pod_autoscaler_enabled = true } + + lifecycle { + ignore_changes = [ microsoft_defender ] + } } resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf index 5e0a3a3ee..bf65750a4 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf @@ -11,20 +11,15 @@ resource "azurerm_private_dns_zone_virtual_network_link" "this" { } resource "azurerm_private_endpoint" "this" { - name = var.endpoint + name = var.name location = var.location resource_group_name = var.resource_group_name subnet_id = var.subnet_id private_service_connection { - name = "${var.name}Connection" + name = "connection" private_connection_resource_id = var.private_connection_resource_id is_manual_connection = false subresource_names = [var.subresource_name] } - - private_dns_zone_group { - name = "${var.name}Group" - private_dns_zone_ids = [azurerm_private_dns_zone.this.id] - } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf index 42557e820..8933b684a 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf @@ -10,10 +10,6 @@ variable "resource_group_name" { type = string } -variable "endpoint" { - type = string -} - variable "subresource_name" { type = string } diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 55085a5e0..4526d773b 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -9,8 +9,8 @@ variable "location" { } variable "openai_subdomain" { - type = string - default = "" + type = string + default = "magic8ball" } variable "kubernetes_version" { From d25cdfa6ac3c3d7146ff5fea620243019d85a15b Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 06:03:47 -0500 Subject: [PATCH 059/119] Fixes --- scenarios/AksOpenAiTerraform/terraform/main.tf | 10 +++++----- .../AksOpenAiTerraform/terraform/modules/aks/main.tf | 2 +- .../terraform/modules/virtual_network/variables.tf | 11 +++++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index c226a0923..60a87a7e7 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -290,11 +290,11 @@ resource "azurerm_user_assigned_identity" "aks_workload_identity" { resource "azurerm_federated_identity_credential" "federated_identity_credential" { name = "${title(local.namespace)}FederatedIdentity" resource_group_name = azurerm_resource_group.rg.name - - audience = ["api://AzureADTokenExchange"] - issuer = module.aks_cluster.oidc_issuer_url - parent_id = azurerm_user_assigned_identity.aks_workload_identity.id - subject = "system:serviceaccount:${local.namespace}:${local.service_account_name}" + + audience = ["api://AzureADTokenExchange"] + issuer = module.aks_cluster.oidc_issuer_url + parent_id = azurerm_user_assigned_identity.aks_workload_identity.id + subject = "system:serviceaccount:${local.namespace}:${local.service_account_name}" } resource "azurerm_role_assignment" "cognitive_services_user_assignment" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index fdcd693e0..b2b77ecb4 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -66,7 +66,7 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { } lifecycle { - ignore_changes = [ microsoft_defender ] + ignore_changes = [microsoft_defender] } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf index c4a844fcb..fcadabecf 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf @@ -19,10 +19,13 @@ variable "subnets" { type = list(object({ name = string address_prefixes = list(string) - delegation = object({ name = string, service_delegation = object({ - name = string - actions = list(string) - }) }) + delegation = optional(object({ + name = string, + service_delegation = object({ + name = string + actions = list(string) + }) + })) })) } From d7d9be284ea5dee4449283ad02f978561ea36698 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 06:37:52 -0500 Subject: [PATCH 060/119] Clean --- .../AksOpenAiTerraform/terraform/main.tf | 54 +++++-------------- .../modules/bastion_host/variables.tf | 10 ++-- .../modules/container_registry/main.tf | 14 ----- .../modules/container_registry/variables.tf | 4 -- .../terraform/modules/key_vault/main.tf | 16 +++--- .../terraform/modules/key_vault/variables.tf | 40 +------------- .../terraform/modules/log_analytics/main.tf | 16 +++--- .../modules/log_analytics/variables.tf | 4 -- .../terraform/modules/openai/main.tf | 2 +- .../terraform/modules/openai/variables.tf | 13 +---- .../terraform/modules/storage_account/main.tf | 14 ++--- .../modules/storage_account/variables.tf | 18 +------ .../AksOpenAiTerraform/terraform/variables.tf | 14 ++--- 13 files changed, 44 insertions(+), 175 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 60a87a7e7..dfae41848 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -37,9 +37,6 @@ locals { namespace = "magic8ball" service_account_name = "magic8ball-sa" - - log_analytics_workspace_name = "Workspace" - log_analytics_retention_days = 30 } resource "azurerm_resource_group" "rg" { @@ -70,11 +67,9 @@ module "openai" { } } ] - custom_subdomain_name = var.openai_subdomain - public_network_access_enabled = true + custom_subdomain_name = var.openai_subdomain - log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = local.log_analytics_retention_days + log_analytics_workspace_id = module.log_analytics_workspace.id } module "aks_cluster" { @@ -87,8 +82,8 @@ module "aks_cluster" { kubernetes_version = var.kubernetes_version sku_tier = "Free" - system_node_pool_vm_size = var.system_node_pool_vm_size - user_node_pool_vm_size = var.user_node_pool_vm_size + system_node_pool_vm_size = "Standard_D8ds_v5" + user_node_pool_vm_size = "Standard_D8ds_v5" system_node_pool_subnet_id = module.virtual_network.subnet_ids["SystemSubnet"] user_node_pool_subnet_id = module.virtual_network.subnet_ids["UserSubnet"] @@ -105,8 +100,7 @@ module "container_registry" { location = var.location resource_group_name = azurerm_resource_group.rg.name - sku = "Premium" - admin_enabled = true + sku = "Premium" log_analytics_workspace_id = module.log_analytics_workspace.id } @@ -116,11 +110,6 @@ module "storage_account" { name = "boot${random_string.storage_account_suffix.result}" location = var.location resource_group_name = azurerm_resource_group.rg.name - - account_kind = "StorageV2" - account_tier = "Standard" - replication_type = "LRS" - is_hns_enabled = false } module "key_vault" { @@ -129,34 +118,20 @@ module "key_vault" { location = var.location resource_group_name = azurerm_resource_group.rg.name - tenant_id = local.tenant_id - sku_name = "standard" - enabled_for_deployment = true - enabled_for_disk_encryption = true - enabled_for_template_deployment = true - enable_rbac_authorization = true - purge_protection_enabled = false - soft_delete_retention_days = 30 - bypass = "AzureServices" - default_action = "Allow" - log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = local.log_analytics_retention_days + tenant_id = local.tenant_id + sku_name = "standard" + + log_analytics_workspace_id = module.log_analytics_workspace.id } module "log_analytics_workspace" { source = "./modules/log_analytics" - name = local.log_analytics_workspace_name + name = "Workspace" location = var.location resource_group_name = azurerm_resource_group.rg.name sku = "PerGB2018" - retention_in_days = local.log_analytics_retention_days - solution_plan_map = { - ContainerInsights = { - product = "OMSGallery/ContainerInsights" - publisher = "Microsoft" - } - } + retention_in_days = 30 } ############################################################################### @@ -219,8 +194,7 @@ module "bastion_host" { subnet_id = module.virtual_network.subnet_ids["AzureBastionSubnet"] - log_analytics_workspace_id = module.log_analytics_workspace.id - log_analytics_retention_days = local.log_analytics_retention_days + log_analytics_workspace_id = module.log_analytics_workspace.id } ############################################################################### @@ -234,7 +208,6 @@ module "acr_private_dns_zone" { name = "privatelink.azurecr.io" subresource_name = "account" private_connection_resource_id = module.openai.id - virtual_network_id = module.virtual_network.id subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } @@ -247,7 +220,6 @@ module "openai_private_dns_zone" { name = "privatelink.openai.azure.com" subresource_name = "registry" private_connection_resource_id = module.container_registry.id - virtual_network_id = module.virtual_network.id subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } @@ -260,7 +232,6 @@ module "key_vault_private_dns_zone" { name = "privatelink.vaultcore.azure.net" subresource_name = "vault" private_connection_resource_id = module.key_vault.id - virtual_network_id = module.virtual_network.id subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } @@ -273,7 +244,6 @@ module "blob_private_dns_zone" { name = "privatelink.blob.core.windows.net" subresource_name = "blob" private_connection_resource_id = module.storage_account.id - virtual_network_id = module.virtual_network.id subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf index ab2e33027..c3b2d0b5d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf @@ -1,12 +1,12 @@ -variable "resource_group_name" { +variable "name" { type = string } -variable "name" { +variable "location" { type = string } -variable "location" { +variable "resource_group_name" { type = string } @@ -16,8 +16,4 @@ variable "subnet_id" { variable "log_analytics_workspace_id" { type = string -} - -variable "log_analytics_retention_days" { - type = number } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf index 52e65bc5d..d071ad376 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf @@ -3,20 +3,6 @@ resource "azurerm_container_registry" "acr" { resource_group_name = var.resource_group_name location = var.location sku = var.sku - admin_enabled = var.admin_enabled - - identity { - type = "UserAssigned" - identity_ids = [ - azurerm_user_assigned_identity.acr_identity.id - ] - } -} - -resource "azurerm_user_assigned_identity" "acr_identity" { - name = "${var.name}Identity" - resource_group_name = var.resource_group_name - location = var.location } resource "azurerm_monitor_diagnostic_setting" "settings" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf index bf4616efb..df252b035 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf @@ -10,10 +10,6 @@ variable "location" { type = string } -variable "admin_enabled" { - type = string -} - variable "sku" { type = string } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf index aab17f34b..94c357af1 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf @@ -5,20 +5,20 @@ resource "azurerm_key_vault" "key_vault" { tenant_id = var.tenant_id sku_name = var.sku_name - enabled_for_deployment = var.enabled_for_deployment - enabled_for_disk_encryption = var.enabled_for_disk_encryption - enabled_for_template_deployment = var.enabled_for_template_deployment - enable_rbac_authorization = var.enable_rbac_authorization - purge_protection_enabled = var.purge_protection_enabled - soft_delete_retention_days = var.soft_delete_retention_days + enabled_for_deployment = true + enabled_for_disk_encryption = true + enabled_for_template_deployment = true + enable_rbac_authorization = true + purge_protection_enabled = false + soft_delete_retention_days = 30 timeouts { delete = "60m" } network_acls { - bypass = var.bypass - default_action = var.default_action + bypass = "AzureServices" + default_action = "Allow" } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf index 3421eb126..2918ab083 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf @@ -2,11 +2,11 @@ variable "name" { type = string } -variable "resource_group_name" { +variable "location" { type = string } -variable "location" { +variable "resource_group_name" { type = string } @@ -18,42 +18,6 @@ variable "sku_name" { type = string } -variable "enabled_for_deployment" { - type = bool -} - -variable "enabled_for_disk_encryption" { - type = bool -} - -variable "enabled_for_template_deployment" { - type = bool -} - -variable "enable_rbac_authorization" { - type = bool -} - -variable "purge_protection_enabled" { - type = bool -} - -variable "soft_delete_retention_days" { - type = number -} - -variable "bypass" { - type = string -} - -variable "default_action" { - type = string -} - variable "log_analytics_workspace_id" { type = string -} - -variable "log_analytics_retention_days" { - type = number } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf index 5f2bfe48d..e3c50d5d5 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf @@ -1,4 +1,4 @@ -resource "azurerm_log_analytics_workspace" "log_analytics_workspace" { +resource "azurerm_log_analytics_workspace" "this" { name = var.name location = var.location resource_group_name = var.resource_group_name @@ -6,17 +6,15 @@ resource "azurerm_log_analytics_workspace" "log_analytics_workspace" { retention_in_days = var.retention_in_days } -resource "azurerm_log_analytics_solution" "la_solution" { - for_each = var.solution_plan_map - - solution_name = each.key +resource "azurerm_log_analytics_solution" "this" { + solution_name = "ContainerInsights" location = var.location resource_group_name = var.resource_group_name - workspace_resource_id = azurerm_log_analytics_workspace.log_analytics_workspace.id - workspace_name = azurerm_log_analytics_workspace.log_analytics_workspace.name + workspace_resource_id = azurerm_log_analytics_workspace.this.id + workspace_name = azurerm_log_analytics_workspace.this.name plan { - product = each.value.product - publisher = each.value.publisher + product = "OMSGallery/ContainerInsights" + publisher = "Microsoft" } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf index 6a0d04469..9c1aa1f04 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf @@ -14,10 +14,6 @@ variable "sku" { type = string } -variable "solution_plan_map" { - type = map(any) -} - variable "retention_in_days" { type = number } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf index 3b2964d0f..8821e5ce6 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf @@ -6,7 +6,7 @@ resource "azurerm_cognitive_account" "openai" { kind = "OpenAI" custom_subdomain_name = var.custom_subdomain_name sku_name = var.sku_name - public_network_access_enabled = var.public_network_access_enabled + public_network_access_enabled = true identity { type = "SystemAssigned" diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf index 9bb21252d..2eee76ed2 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf @@ -1,4 +1,4 @@ -variable "resource_group_name" { +variable "name" { type = string } @@ -6,7 +6,7 @@ variable "location" { type = string } -variable "name" { +variable "resource_group_name" { type = string } @@ -18,11 +18,6 @@ variable "custom_subdomain_name" { type = string } -variable "public_network_access_enabled" { - type = bool - default = true -} - variable "deployments" { type = list(object({ name = string @@ -35,8 +30,4 @@ variable "deployments" { variable "log_analytics_workspace_id" { type = string -} - -variable "log_analytics_retention_days" { - type = number } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf index 6e885b845..7d265fa25 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf @@ -1,16 +1,12 @@ resource "azurerm_storage_account" "storage_account" { name = var.name + location = var.location resource_group_name = var.resource_group_name - location = var.location - account_kind = var.account_kind - account_tier = var.account_tier - account_replication_type = var.replication_type - is_hns_enabled = var.is_hns_enabled + account_kind = "StorageV2" + account_tier = "Standard" + account_replication_type = "LRS" + is_hns_enabled = false allow_nested_items_to_be_public = false - - identity { - type = "SystemAssigned" - } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf index dbd9d37c6..3d2c4d24d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf @@ -1,7 +1,3 @@ -variable "resource_group_name" { - type = string -} - variable "name" { type = string } @@ -10,18 +6,6 @@ variable "location" { type = string } -variable "account_kind" { - type = string -} - -variable "account_tier" { - type = string -} - -variable "replication_type" { +variable "resource_group_name" { type = string -} - -variable "is_hns_enabled" { - type = bool } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 4526d773b..ca6f361de 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -13,22 +13,14 @@ variable "openai_subdomain" { default = "magic8ball" } +# -test465544 + variable "kubernetes_version" { type = string default = "1.30.7" } -variable "system_node_pool_vm_size" { - type = string - default = "Standard_D8ds_v5" -} - -variable "user_node_pool_vm_size" { - type = string - default = "Standard_D8ds_v5" -} - variable "email" { type = string - default = "paolos@microsoft.com" + default = "ariaamini@microsoft.com" } \ No newline at end of file From 97e0bf625c9a2b156c07879f6e8c3dd6e7b486ae Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 06:45:27 -0500 Subject: [PATCH 061/119] Clean --- .../AksOpenAiTerraform/terraform/main.tf | 16 +--- .../modules/diagnostic_setting/main.tf | 12 --- .../modules/diagnostic_setting/variables.tf | 79 ------------------- .../terraform/modules/log_analytics/output.tf | 31 +------- .../AksOpenAiTerraform/terraform/provider.tf | 12 +++ 5 files changed, 17 insertions(+), 133 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf create mode 100644 scenarios/AksOpenAiTerraform/terraform/provider.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index dfae41848..14127733f 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -1,16 +1,3 @@ -terraform { - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "~> 4.16.0" - } - } -} - -provider "azurerm" { - features {} -} - data "azurerm_client_config" "current" { } @@ -39,6 +26,9 @@ locals { service_account_name = "magic8ball-sa" } +############################################################################### +# Resource Group +############################################################################### resource "azurerm_resource_group" "rg" { name = "${var.resource_group_name_prefix}-${local.random_id}-rg" location = var.location diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf deleted file mode 100644 index c188cf7ac..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/main.tf +++ /dev/null @@ -1,12 +0,0 @@ -resource "azurerm_monitor_diagnostic_setting" "settings" { - name = var.name - target_resource_id = var.target_resource_id - - log_analytics_workspace_id = var.log_analytics_workspace_id - log_analytics_destination_type = var.log_analytics_destination_type - - eventhub_name = var.eventhub_name - eventhub_authorization_rule_id = var.eventhub_authorization_rule_id - - storage_account_id = var.storage_account_id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf deleted file mode 100644 index 7165884e9..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/diagnostic_setting/variables.tf +++ /dev/null @@ -1,79 +0,0 @@ - -variable "name" { - description = "(Required) Specifies the name of the Container Registry. Changing this forces a new resource to be created." - type = string -} - -variable "resource_group_name" { - description = "(Required) The name of the resource group in which to create the Container Registry. Changing this forces a new resource to be created." - type = string -} - -variable "location" { - description = "(Required) Specifies the supported Azure location where the resource exists. Changing this forces a new resource to be created." - type = string -} - -variable "retention_policy_enabled" { - description = "(Required) Is this Retention Policy enabled?" - type = bool - default = true -} - -variable "retention_policy_days" { - description = "(Optional) The number of days for which this Retention Policy should apply." - type = number - default = 30 -} - -variable "target_resource_id" { - description = "(Required) The ID of an existing Resource on which to configure Diagnostic Settings. Changing this forces a new resource to be created." - type = string -} - -variable "log_analytics_workspace_id" { - description = "(Optional) Specifies the ID of a Log Analytics Workspace where Diagnostics Data should be sent." - type = string -} - -variable "log_analytics_destination_type" { - description = "(Optional) When set to 'Dedicated' logs sent to a Log Analytics workspace will go into resource specific tables, instead of the legacy AzureDiagnostics table." - type = string - default = null -} - -variable "storage_account_id" { - description = "(Optional) The ID of the Storage Account where logs should be sent. Changing this forces a new resource to be created." - type = string - default = null -} - -variable "eventhub_name" { - description = "(Optional) Specifies the name of the Event Hub where Diagnostics Data should be sent. Changing this forces a new resource to be created." - type = string - default = null -} - -variable "eventhub_authorization_rule_id" { - description = "(Optional) Specifies the ID of an Event Hub Namespace Authorization Rule used to send Diagnostics Data. Changing this forces a new resource to be created." - type = string - default = null -} - -variable "logs" { - description = "(Optional) Specifies a list of log categories to enable." - type = list(string) - default = [] -} - -variable "metrics" { - description = "(Optional) Specifies a list of metrics to enable." - type = list(string) - default = [] -} - -variable "tags" { - description = "(Optional) A mapping of tags to assign to the resource." - type = map(any) - default = {} -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf index 7abcf881f..837cd9e49 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf @@ -1,30 +1,3 @@ output "id" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.id - description = "Specifies the resource id of the log analytics workspace" -} - -output "location" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.location - description = "Specifies the location of the log analytics workspace" -} - -output "name" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.name - description = "Specifies the name of the log analytics workspace" -} - -output "resource_group_name" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.resource_group_name - description = "Specifies the name of the resource group that contains the log analytics workspace" -} - -output "workspace_id" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.workspace_id - description = "Specifies the workspace id of the log analytics workspace" -} - -output "primary_shared_key" { - value = azurerm_log_analytics_workspace.log_analytics_workspace.primary_shared_key - description = "Specifies the workspace key of the log analytics workspace" - sensitive = true -} + value = azurerm_log_analytics_workspace.this.id +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/provider.tf b/scenarios/AksOpenAiTerraform/terraform/provider.tf new file mode 100644 index 000000000..5d9512e59 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/provider.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.16.0" + } + } +} + +provider "azurerm" { + features {} +} \ No newline at end of file From bae1765ccc1e7a77a33f5af4b7666639e45ad9e9 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 06:47:54 -0500 Subject: [PATCH 062/119] Remove old files --- scenarios/AksOpenAiTerraform/plan.txt | 1116 ----------------- scenarios/AksOpenAiTerraform/run.sh | 99 -- scenarios/AksOpenAiTerraform/scripts/.env | 2 - .../scripts/00-variables.sh | 71 -- .../scripts/01-build-docker-image.sh | 12 - .../{wip => scripts}/01-push-app-image.sh | 0 .../scripts/02-run-docker-container.sh | 21 - .../scripts/03-push-docker-image.sh | 16 - .../04-create-nginx-ingress-controller.sh | 3 - .../scripts/05-install-cert-manager.sh | 3 - .../scripts/06-create-cluster-issuer.sh | 3 - .../scripts/09-deploy-app.sh | 9 - .../scripts/10-create-ingress.sh | 3 - .../scripts/{ => app}/Dockerfile | 0 .../scripts/{ => app}/app.py | 0 .../scripts/{ => app}/images/magic8ball.png | Bin .../scripts/{ => app}/images/robot.png | Bin .../scripts/{ => app}/requirements.txt | 0 .../install-nginx-via-helm-and-create-sa.sh | 0 .../{ => manifests}/cluster-issuer.yml | 0 .../scripts/{ => manifests}/configMap.yml | 0 .../scripts/{ => manifests}/deployment.yml | 0 .../scripts/{ => manifests}/ingress.yml | 0 .../scripts/{ => manifests}/service.yml | 0 .../AksOpenAiTerraform/terraform/variables.tf | 2 +- .../wip/04-create-nginx-ingress-controller.sh | 36 - .../wip/05-install-cert-manager.sh | 31 - .../wip/06-create-cluster-issuer.sh | 16 - .../07-create-workload-managed-identity.sh | 104 -- .../wip/08-create-service-account.sh | 103 -- .../AksOpenAiTerraform/wip/09-deploy-app.sh | 37 - .../wip/10-create-ingress.sh | 9 - .../wip/11-configure-dns.sh | 79 -- .../AksOpenAiTerraform/wip/app/Dockerfile | 94 -- scenarios/AksOpenAiTerraform/wip/app/app.py | 347 ----- .../wip/app/images/magic8ball.png | Bin 37452 -> 0 bytes .../wip/app/images/robot.png | Bin 1686 -> 0 bytes .../wip/app/requirements.txt | 145 --- .../wip/manifests/cluster-issuer.yml | 18 - .../wip/manifests/configMap.yml | 14 - .../wip/manifests/deployment.yml | 123 -- .../wip/manifests/ingress.yml | 30 - .../wip/manifests/service.yml | 13 - 43 files changed, 1 insertion(+), 2558 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/plan.txt delete mode 100644 scenarios/AksOpenAiTerraform/run.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/.env delete mode 100644 scenarios/AksOpenAiTerraform/scripts/00-variables.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/01-build-docker-image.sh rename scenarios/AksOpenAiTerraform/{wip => scripts}/01-push-app-image.sh (100%) delete mode 100644 scenarios/AksOpenAiTerraform/scripts/02-run-docker-container.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/03-push-docker-image.sh rename scenarios/AksOpenAiTerraform/scripts/{ => app}/Dockerfile (100%) rename scenarios/AksOpenAiTerraform/scripts/{ => app}/app.py (100%) rename scenarios/AksOpenAiTerraform/scripts/{ => app}/images/magic8ball.png (100%) rename scenarios/AksOpenAiTerraform/scripts/{ => app}/images/robot.png (100%) rename scenarios/AksOpenAiTerraform/scripts/{ => app}/requirements.txt (100%) rename scenarios/AksOpenAiTerraform/{wip => scripts}/install-nginx-via-helm-and-create-sa.sh (100%) rename scenarios/AksOpenAiTerraform/scripts/{ => manifests}/cluster-issuer.yml (100%) rename scenarios/AksOpenAiTerraform/scripts/{ => manifests}/configMap.yml (100%) rename scenarios/AksOpenAiTerraform/scripts/{ => manifests}/deployment.yml (100%) rename scenarios/AksOpenAiTerraform/scripts/{ => manifests}/ingress.yml (100%) rename scenarios/AksOpenAiTerraform/scripts/{ => manifests}/service.yml (100%) delete mode 100644 scenarios/AksOpenAiTerraform/wip/04-create-nginx-ingress-controller.sh delete mode 100644 scenarios/AksOpenAiTerraform/wip/05-install-cert-manager.sh delete mode 100644 scenarios/AksOpenAiTerraform/wip/06-create-cluster-issuer.sh delete mode 100644 scenarios/AksOpenAiTerraform/wip/07-create-workload-managed-identity.sh delete mode 100644 scenarios/AksOpenAiTerraform/wip/08-create-service-account.sh delete mode 100644 scenarios/AksOpenAiTerraform/wip/09-deploy-app.sh delete mode 100644 scenarios/AksOpenAiTerraform/wip/10-create-ingress.sh delete mode 100644 scenarios/AksOpenAiTerraform/wip/11-configure-dns.sh delete mode 100644 scenarios/AksOpenAiTerraform/wip/app/Dockerfile delete mode 100644 scenarios/AksOpenAiTerraform/wip/app/app.py delete mode 100644 scenarios/AksOpenAiTerraform/wip/app/images/magic8ball.png delete mode 100644 scenarios/AksOpenAiTerraform/wip/app/images/robot.png delete mode 100644 scenarios/AksOpenAiTerraform/wip/app/requirements.txt delete mode 100644 scenarios/AksOpenAiTerraform/wip/manifests/cluster-issuer.yml delete mode 100644 scenarios/AksOpenAiTerraform/wip/manifests/configMap.yml delete mode 100644 scenarios/AksOpenAiTerraform/wip/manifests/deployment.yml delete mode 100644 scenarios/AksOpenAiTerraform/wip/manifests/ingress.yml delete mode 100644 scenarios/AksOpenAiTerraform/wip/manifests/service.yml diff --git a/scenarios/AksOpenAiTerraform/plan.txt b/scenarios/AksOpenAiTerraform/plan.txt deleted file mode 100644 index aa17b1c49..000000000 --- a/scenarios/AksOpenAiTerraform/plan.txt +++ /dev/null @@ -1,1116 +0,0 @@ -Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the -following symbols: - + create - -Terraform will perform the following actions: - - # azurerm_federated_identity_credential.federated_identity_credential will be created - + resource "azurerm_federated_identity_credential" "federated_identity_credential" { - + audience = [ - + "api://AzureADTokenExchange", - ] - + id = (known after apply) - + issuer = (known after apply) - + name = "Magic8ballFederatedIdentity" - + parent_id = (known after apply) - + resource_group_name = (known after apply) - + subject = "system:serviceaccount:magic8ball:magic8ball-sa" - } - - # azurerm_resource_group.rg will be created - + resource "azurerm_resource_group" "rg" { - + id = (known after apply) - + location = "westus3" - + name = (known after apply) - } - - # azurerm_role_assignment.acr_pull_assignment will be created - + resource "azurerm_role_assignment" "acr_pull_assignment" { - + condition_version = (known after apply) - + id = (known after apply) - + name = (known after apply) - + principal_id = (known after apply) - + principal_type = (known after apply) - + role_definition_id = (known after apply) - + role_definition_name = "AcrPull" - + scope = (known after apply) - + skip_service_principal_aad_check = (known after apply) - } - - # azurerm_role_assignment.cognitive_services_user_assignment will be created - + resource "azurerm_role_assignment" "cognitive_services_user_assignment" { - + condition_version = (known after apply) - + id = (known after apply) - + name = (known after apply) - + principal_id = (known after apply) - + principal_type = (known after apply) - + role_definition_id = (known after apply) - + role_definition_name = "Cognitive Services User" - + scope = (known after apply) - + skip_service_principal_aad_check = (known after apply) - } - - # azurerm_role_assignment.network_contributor_assignment will be created - + resource "azurerm_role_assignment" "network_contributor_assignment" { - + condition_version = (known after apply) - + id = (known after apply) - + name = (known after apply) - + principal_id = (known after apply) - + principal_type = (known after apply) - + role_definition_id = (known after apply) - + role_definition_name = "Network Contributor" - + scope = (known after apply) - + skip_service_principal_aad_check = (known after apply) - } - - # azurerm_user_assigned_identity.aks_workload_identity will be created - + resource "azurerm_user_assigned_identity" "aks_workload_identity" { - + client_id = (known after apply) - + id = (known after apply) - + location = "westus3" - + name = "WorkloadManagedIdentity" - + principal_id = (known after apply) - + resource_group_name = (known after apply) - + tenant_id = (known after apply) - } - - # random_string.rg_suffix will be created - + resource "random_string" "rg_suffix" { - + id = (known after apply) - + length = 6 - + lower = false - + min_lower = 0 - + min_numeric = 0 - + min_special = 0 - + min_upper = 0 - + number = true - + numeric = true - + result = (known after apply) - + special = false - + upper = false - } - - # random_string.storage_account_suffix will be created - + resource "random_string" "storage_account_suffix" { - + id = (known after apply) - + length = 8 - + lower = true - + min_lower = 0 - + min_numeric = 0 - + min_special = 0 - + min_upper = 0 - + number = false - + numeric = false - + result = (known after apply) - + special = false - + upper = false - } - - # module.acr_private_dns_zone.azurerm_private_dns_zone.private_dns_zone will be created - + resource "azurerm_private_dns_zone" "private_dns_zone" { - + id = (known after apply) - + max_number_of_record_sets = (known after apply) - + max_number_of_virtual_network_links = (known after apply) - + max_number_of_virtual_network_links_with_registration = (known after apply) - + name = "privatelink.azurecr.io" - + number_of_record_sets = (known after apply) - + resource_group_name = (known after apply) - - + soa_record (known after apply) - } - - # module.acr_private_dns_zone.azurerm_private_dns_zone_virtual_network_link.link["AksVNet"] will be created - + resource "azurerm_private_dns_zone_virtual_network_link" "link" { - + id = (known after apply) - + name = "link_to_aksvnet" - + private_dns_zone_name = "privatelink.azurecr.io" - + registration_enabled = false - + resource_group_name = (known after apply) - + virtual_network_id = (known after apply) - } - - # module.acr_private_endpoint.azurerm_private_endpoint.private_endpoint will be created - + resource "azurerm_private_endpoint" "private_endpoint" { - + custom_dns_configs = (known after apply) - + id = (known after apply) - + location = "westus3" - + name = "AcrPrivateEndpoint" - + network_interface = (known after apply) - + private_dns_zone_configs = (known after apply) - + resource_group_name = (known after apply) - + subnet_id = (known after apply) - - + private_dns_zone_group { - + id = (known after apply) - + name = "AcrPrivateDnsZoneGroup" - + private_dns_zone_ids = (known after apply) - } - - + private_service_connection { - + is_manual_connection = false - + name = "AcrPrivateEndpointConnection" - + private_connection_resource_id = (known after apply) - + private_ip_address = (known after apply) - + subresource_names = [ - + "registry", - ] - } - } - - # module.aks_cluster.azurerm_kubernetes_cluster.aks_cluster will be created - + resource "azurerm_kubernetes_cluster" "aks_cluster" { - + automatic_upgrade_channel = "stable" - + azure_policy_enabled = true - + current_kubernetes_version = (known after apply) - + dns_prefix = "akscluster" - + fqdn = (known after apply) - + http_application_routing_enabled = false - + http_application_routing_zone_name = (known after apply) - + id = (known after apply) - + image_cleaner_enabled = true - + image_cleaner_interval_hours = 72 - + kube_admin_config = (sensitive value) - + kube_admin_config_raw = (sensitive value) - + kube_config = (sensitive value) - + kube_config_raw = (sensitive value) - + kubernetes_version = "1.30.7" - + location = "westus3" - + name = "AksCluster" - + node_os_upgrade_channel = "NodeImage" - + node_resource_group = (known after apply) - + node_resource_group_id = (known after apply) - + oidc_issuer_enabled = true - + oidc_issuer_url = (known after apply) - + open_service_mesh_enabled = true - + portal_fqdn = (known after apply) - + private_cluster_enabled = false - + private_cluster_public_fqdn_enabled = false - + private_dns_zone_id = (known after apply) - + private_fqdn = (known after apply) - + resource_group_name = (known after apply) - + role_based_access_control_enabled = true - + run_command_enabled = true - + sku_tier = "Free" - + support_plan = "KubernetesOfficial" - + workload_identity_enabled = true - - + auto_scaler_profile (known after apply) - - + azure_active_directory_role_based_access_control { - + azure_rbac_enabled = true - + tenant_id = "72f988bf-86f1-41af-91ab-2d7cd011db47" - } - - + default_node_pool { - + kubelet_disk_type = (known after apply) - + max_pods = 50 - + name = "system" - + node_count = 1 - + node_labels = (known after apply) - + orchestrator_version = (known after apply) - + os_disk_size_gb = (known after apply) - + os_disk_type = "Ephemeral" - + os_sku = (known after apply) - + pod_subnet_id = (known after apply) - + scale_down_mode = "Delete" - + type = "VirtualMachineScaleSets" - + ultra_ssd_enabled = false - + vm_size = "Standard_D8ds_v5" - + vnet_subnet_id = (known after apply) - + workload_runtime = (known after apply) - + zones = [ - + "1", - + "2", - + "3", - ] - - + upgrade_settings { - + drain_timeout_in_minutes = 0 - + max_surge = "10%" - + node_soak_duration_in_minutes = 0 - } - } - - + identity { - + identity_ids = (known after apply) - + principal_id = (known after apply) - + tenant_id = (known after apply) - + type = "UserAssigned" - } - - + kubelet_identity (known after apply) - - + network_profile { - + dns_service_ip = "10.2.0.10" - + ip_versions = (known after apply) - + load_balancer_sku = "standard" - + network_data_plane = "azure" - + network_mode = (known after apply) - + network_plugin = "azure" - + network_policy = (known after apply) - + outbound_type = "userAssignedNATGateway" - + pod_cidr = (known after apply) - + pod_cidrs = (known after apply) - + service_cidr = "10.2.0.0/24" - + service_cidrs = (known after apply) - - + load_balancer_profile (known after apply) - - + nat_gateway_profile (known after apply) - } - - + oms_agent { - + log_analytics_workspace_id = (known after apply) - + msi_auth_for_monitoring_enabled = true - + oms_agent_identity = (known after apply) - } - - + windows_profile (known after apply) - - + workload_autoscaler_profile { - + keda_enabled = true - + vertical_pod_autoscaler_enabled = true - } - } - - # module.aks_cluster.azurerm_kubernetes_cluster_node_pool.node_pool will be created - + resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { - + id = (known after apply) - + kubelet_disk_type = (known after apply) - + kubernetes_cluster_id = (known after apply) - + max_pods = 50 - + mode = "User" - + name = "user" - + node_count = (known after apply) - + node_labels = (known after apply) - + orchestrator_version = "1.30.7" - + os_disk_size_gb = (known after apply) - + os_disk_type = "Ephemeral" - + os_sku = (known after apply) - + os_type = "Linux" - + pod_subnet_id = (known after apply) - + priority = "Regular" - + scale_down_mode = "Delete" - + spot_max_price = -1 - + ultra_ssd_enabled = false - + vm_size = "Standard_D8ds_v5" - + vnet_subnet_id = (known after apply) - + zones = [ - + "1", - + "2", - + "3", - ] - } - - # module.aks_cluster.azurerm_monitor_diagnostic_setting.settings will be created - + resource "azurerm_monitor_diagnostic_setting" "settings" { - + id = (known after apply) - + log_analytics_destination_type = (known after apply) - + log_analytics_workspace_id = (known after apply) - + name = "AksDiagnosticsSettings" - + target_resource_id = (known after apply) - - + enabled_log { - + category = "cluster-autoscaler" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "guard" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "kube-apiserver" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "kube-audit" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "kube-audit-admin" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "kube-controller-manager" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "kube-scheduler" - # (1 unchanged attribute hidden) - } - - + metric { - + category = "AllMetrics" - + enabled = true - } - } - - # module.aks_cluster.azurerm_user_assigned_identity.aks_identity will be created - + resource "azurerm_user_assigned_identity" "aks_identity" { - + client_id = (known after apply) - + id = (known after apply) - + location = "westus3" - + name = "AksClusterIdentity" - + principal_id = (known after apply) - + resource_group_name = (known after apply) - + tenant_id = (known after apply) - } - - # module.bastion_host.azurerm_bastion_host.bastion_host will be created - + resource "azurerm_bastion_host" "bastion_host" { - + copy_paste_enabled = true - + dns_name = (known after apply) - + file_copy_enabled = false - + id = (known after apply) - + ip_connect_enabled = false - + kerberos_enabled = false - + location = "westus3" - + name = "BastionHost" - + resource_group_name = (known after apply) - + scale_units = 2 - + session_recording_enabled = false - + shareable_link_enabled = false - + sku = "Basic" - + tunneling_enabled = false - - + ip_configuration { - + name = "configuration" - + public_ip_address_id = (known after apply) - + subnet_id = (known after apply) - } - } - - # module.bastion_host.azurerm_monitor_diagnostic_setting.pip_settings will be created - + resource "azurerm_monitor_diagnostic_setting" "pip_settings" { - + id = (known after apply) - + log_analytics_destination_type = (known after apply) - + log_analytics_workspace_id = (known after apply) - + name = "BastionDdosDiagnosticsSettings" - + target_resource_id = (known after apply) - - + enabled_log { - + category = "DDoSMitigationFlowLogs" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "DDoSMitigationReports" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "DDoSProtectionNotifications" - # (1 unchanged attribute hidden) - } - - + metric { - + category = "AllMetrics" - + enabled = true - } - } - - # module.bastion_host.azurerm_monitor_diagnostic_setting.settings will be created - + resource "azurerm_monitor_diagnostic_setting" "settings" { - + id = (known after apply) - + log_analytics_destination_type = (known after apply) - + log_analytics_workspace_id = (known after apply) - + name = "BastionDiagnosticsSettings" - + target_resource_id = (known after apply) - - + enabled_log { - + category = "BastionAuditLogs" - # (1 unchanged attribute hidden) - } - - + metric { - + category = "AllMetrics" - + enabled = true - } - } - - # module.bastion_host.azurerm_public_ip.public_ip will be created - + resource "azurerm_public_ip" "public_ip" { - + allocation_method = "Static" - + ddos_protection_mode = "VirtualNetworkInherited" - + fqdn = (known after apply) - + id = (known after apply) - + idle_timeout_in_minutes = 4 - + ip_address = (known after apply) - + ip_version = "IPv4" - + location = "westus3" - + name = "BastionHostPublicIp" - + resource_group_name = (known after apply) - + sku = "Standard" - + sku_tier = "Regional" - } - - # module.blob_private_dns_zone.azurerm_private_dns_zone.private_dns_zone will be created - + resource "azurerm_private_dns_zone" "private_dns_zone" { - + id = (known after apply) - + max_number_of_record_sets = (known after apply) - + max_number_of_virtual_network_links = (known after apply) - + max_number_of_virtual_network_links_with_registration = (known after apply) - + name = "privatelink.blob.core.windows.net" - + number_of_record_sets = (known after apply) - + resource_group_name = (known after apply) - - + soa_record (known after apply) - } - - # module.blob_private_dns_zone.azurerm_private_dns_zone_virtual_network_link.link["AksVNet"] will be created - + resource "azurerm_private_dns_zone_virtual_network_link" "link" { - + id = (known after apply) - + name = "link_to_aksvnet" - + private_dns_zone_name = "privatelink.blob.core.windows.net" - + registration_enabled = false - + resource_group_name = (known after apply) - + virtual_network_id = (known after apply) - } - - # module.blob_private_endpoint.azurerm_private_endpoint.private_endpoint will be created - + resource "azurerm_private_endpoint" "private_endpoint" { - + custom_dns_configs = (known after apply) - + id = (known after apply) - + location = "westus3" - + name = "BlobStoragePrivateEndpoint" - + network_interface = (known after apply) - + private_dns_zone_configs = (known after apply) - + resource_group_name = (known after apply) - + subnet_id = (known after apply) - - + private_dns_zone_group { - + id = (known after apply) - + name = "BlobPrivateDnsZoneGroup" - + private_dns_zone_ids = (known after apply) - } - - + private_service_connection { - + is_manual_connection = false - + name = "BlobStoragePrivateEndpointConnection" - + private_connection_resource_id = (known after apply) - + private_ip_address = (known after apply) - + subresource_names = [ - + "blob", - ] - } - } - - # module.container_registry.azurerm_container_registry.acr will be created - + resource "azurerm_container_registry" "acr" { - + admin_enabled = true - + admin_password = (sensitive value) - + admin_username = (known after apply) - + encryption = (known after apply) - + export_policy_enabled = true - + id = (known after apply) - + location = "westus3" - + login_server = (known after apply) - + name = (known after apply) - + network_rule_bypass_option = "AzureServices" - + network_rule_set = (known after apply) - + public_network_access_enabled = true - + resource_group_name = (known after apply) - + sku = "Premium" - + trust_policy_enabled = false - + zone_redundancy_enabled = false - - + identity { - + identity_ids = (known after apply) - + principal_id = (known after apply) - + tenant_id = (known after apply) - + type = "UserAssigned" - } - } - - # module.container_registry.azurerm_monitor_diagnostic_setting.settings will be created - + resource "azurerm_monitor_diagnostic_setting" "settings" { - + id = (known after apply) - + log_analytics_destination_type = (known after apply) - + log_analytics_workspace_id = (known after apply) - + name = "ContainerDiagnosticsSettings" - + target_resource_id = (known after apply) - - + enabled_log { - + category = "ContainerRegistryLoginEvents" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "ContainerRegistryRepositoryEvents" - # (1 unchanged attribute hidden) - } - - + metric { - + category = "AllMetrics" - + enabled = true - } - } - - # module.container_registry.azurerm_user_assigned_identity.acr_identity will be created - + resource "azurerm_user_assigned_identity" "acr_identity" { - + client_id = (known after apply) - + id = (known after apply) - + location = "westus3" - + name = (known after apply) - + principal_id = (known after apply) - + resource_group_name = (known after apply) - + tenant_id = (known after apply) - } - - # module.key_vault.azurerm_key_vault.key_vault will be created - + resource "azurerm_key_vault" "key_vault" { - + access_policy = (known after apply) - + enable_rbac_authorization = true - + enabled_for_deployment = true - + enabled_for_disk_encryption = true - + enabled_for_template_deployment = true - + id = (known after apply) - + location = "westus3" - + name = (known after apply) - + public_network_access_enabled = true - + purge_protection_enabled = false - + resource_group_name = (known after apply) - + sku_name = "standard" - + soft_delete_retention_days = 30 - + tenant_id = "72f988bf-86f1-41af-91ab-2d7cd011db47" - + vault_uri = (known after apply) - - + contact (known after apply) - - + network_acls { - + bypass = "AzureServices" - + default_action = "Allow" - } - - + timeouts { - + delete = "60m" - } - } - - # module.key_vault.azurerm_monitor_diagnostic_setting.settings will be created - + resource "azurerm_monitor_diagnostic_setting" "settings" { - + id = (known after apply) - + log_analytics_destination_type = (known after apply) - + log_analytics_workspace_id = (known after apply) - + name = "KeyVaultDiagnosticsSettings" - + target_resource_id = (known after apply) - - + enabled_log { - + category = "AuditEvent" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "AzurePolicyEvaluationDetails" - # (1 unchanged attribute hidden) - } - - + metric { - + category = "AllMetrics" - + enabled = true - } - } - - # module.key_vault_private_dns_zone.azurerm_private_dns_zone.private_dns_zone will be created - + resource "azurerm_private_dns_zone" "private_dns_zone" { - + id = (known after apply) - + max_number_of_record_sets = (known after apply) - + max_number_of_virtual_network_links = (known after apply) - + max_number_of_virtual_network_links_with_registration = (known after apply) - + name = "privatelink.vaultcore.azure.net" - + number_of_record_sets = (known after apply) - + resource_group_name = (known after apply) - - + soa_record (known after apply) - } - - # module.key_vault_private_dns_zone.azurerm_private_dns_zone_virtual_network_link.link["AksVNet"] will be created - + resource "azurerm_private_dns_zone_virtual_network_link" "link" { - + id = (known after apply) - + name = "link_to_aksvnet" - + private_dns_zone_name = "privatelink.vaultcore.azure.net" - + registration_enabled = false - + resource_group_name = (known after apply) - + virtual_network_id = (known after apply) - } - - # module.key_vault_private_endpoint.azurerm_private_endpoint.private_endpoint will be created - + resource "azurerm_private_endpoint" "private_endpoint" { - + custom_dns_configs = (known after apply) - + id = (known after apply) - + location = "westus3" - + name = "VaultPrivateEndpoint" - + network_interface = (known after apply) - + private_dns_zone_configs = (known after apply) - + resource_group_name = (known after apply) - + subnet_id = (known after apply) - - + private_dns_zone_group { - + id = (known after apply) - + name = "KeyVaultPrivateDnsZoneGroup" - + private_dns_zone_ids = (known after apply) - } - - + private_service_connection { - + is_manual_connection = false - + name = "VaultPrivateEndpointConnection" - + private_connection_resource_id = (known after apply) - + private_ip_address = (known after apply) - + subresource_names = [ - + "vault", - ] - } - } - - # module.log_analytics_workspace.azurerm_log_analytics_solution.la_solution["ContainerInsights"] will be created - + resource "azurerm_log_analytics_solution" "la_solution" { - + id = (known after apply) - + location = "westus3" - + resource_group_name = (known after apply) - + solution_name = "ContainerInsights" - + workspace_name = "Workspace" - + workspace_resource_id = (known after apply) - - + plan { - + name = (known after apply) - + product = "OMSGallery/ContainerInsights" - + publisher = "Microsoft" - } - } - - # module.log_analytics_workspace.azurerm_log_analytics_workspace.log_analytics_workspace will be created - + resource "azurerm_log_analytics_workspace" "log_analytics_workspace" { - + allow_resource_only_permissions = true - + daily_quota_gb = -1 - + id = (known after apply) - + internet_ingestion_enabled = true - + internet_query_enabled = true - + local_authentication_disabled = false - + location = "westus3" - + name = "Workspace" - + primary_shared_key = (sensitive value) - + resource_group_name = (known after apply) - + retention_in_days = 30 - + secondary_shared_key = (sensitive value) - + sku = "PerGB2018" - + workspace_id = (known after apply) - } - - # module.nat_gateway.azurerm_nat_gateway.nat_gateway will be created - + resource "azurerm_nat_gateway" "nat_gateway" { - + id = (known after apply) - + idle_timeout_in_minutes = 4 - + location = "westus3" - + name = "NatGateway" - + resource_group_name = (known after apply) - + resource_guid = (known after apply) - + sku_name = "Standard" - + zones = [ - + "1", - ] - } - - # module.nat_gateway.azurerm_nat_gateway_public_ip_association.nat_gategay_public_ip_association will be created - + resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_association" { - + id = (known after apply) - + nat_gateway_id = (known after apply) - + public_ip_address_id = (known after apply) - } - - # module.nat_gateway.azurerm_public_ip.nat_gategay_public_ip will be created - + resource "azurerm_public_ip" "nat_gategay_public_ip" { - + allocation_method = "Static" - + ddos_protection_mode = "VirtualNetworkInherited" - + fqdn = (known after apply) - + id = (known after apply) - + idle_timeout_in_minutes = 4 - + ip_address = (known after apply) - + ip_version = "IPv4" - + location = "westus3" - + name = "NatGatewayPublicIp" - + resource_group_name = (known after apply) - + sku = "Standard" - + sku_tier = "Regional" - + zones = [ - + "1", - ] - } - - # module.nat_gateway.azurerm_subnet_nat_gateway_association.nat-avd-sessionhosts["AzureBastionSubnet"] will be created - + resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { - + id = (known after apply) - + nat_gateway_id = (known after apply) - + subnet_id = (known after apply) - } - - # module.nat_gateway.azurerm_subnet_nat_gateway_association.nat-avd-sessionhosts["PodSubnet"] will be created - + resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { - + id = (known after apply) - + nat_gateway_id = (known after apply) - + subnet_id = (known after apply) - } - - # module.nat_gateway.azurerm_subnet_nat_gateway_association.nat-avd-sessionhosts["SystemSubnet"] will be created - + resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { - + id = (known after apply) - + nat_gateway_id = (known after apply) - + subnet_id = (known after apply) - } - - # module.nat_gateway.azurerm_subnet_nat_gateway_association.nat-avd-sessionhosts["UserSubnet"] will be created - + resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { - + id = (known after apply) - + nat_gateway_id = (known after apply) - + subnet_id = (known after apply) - } - - # module.nat_gateway.azurerm_subnet_nat_gateway_association.nat-avd-sessionhosts["VmSubnet"] will be created - + resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { - + id = (known after apply) - + nat_gateway_id = (known after apply) - + subnet_id = (known after apply) - } - - # module.openai.azurerm_cognitive_account.openai will be created - + resource "azurerm_cognitive_account" "openai" { - + custom_subdomain_name = "magic8ball" - + endpoint = (known after apply) - + id = (known after apply) - + kind = "OpenAI" - + local_auth_enabled = true - + location = "westus3" - + name = (known after apply) - + outbound_network_access_restricted = false - + primary_access_key = (sensitive value) - + public_network_access_enabled = true - + resource_group_name = (known after apply) - + secondary_access_key = (sensitive value) - + sku_name = "S0" - - + identity { - + principal_id = (known after apply) - + tenant_id = (known after apply) - + type = "SystemAssigned" - } - } - - # module.openai.azurerm_cognitive_deployment.deployment["gpt-4"] will be created - + resource "azurerm_cognitive_deployment" "deployment" { - + cognitive_account_id = (known after apply) - + id = (known after apply) - + name = "gpt-4" - + version_upgrade_option = "OnceNewDefaultVersionAvailable" - - + model { - + format = "OpenAI" - + name = "gpt-4" - + version = "turbo-2024-04-09" - } - - + sku { - + capacity = 1 - + name = "Standard" - } - } - - # module.openai.azurerm_monitor_diagnostic_setting.settings will be created - + resource "azurerm_monitor_diagnostic_setting" "settings" { - + id = (known after apply) - + log_analytics_destination_type = (known after apply) - + log_analytics_workspace_id = (known after apply) - + name = "OpenAiDiagnosticsSettings" - + target_resource_id = (known after apply) - - + enabled_log { - + category = "Audit" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "RequestResponse" - # (1 unchanged attribute hidden) - } - + enabled_log { - + category = "Trace" - # (1 unchanged attribute hidden) - } - - + metric { - + category = "AllMetrics" - + enabled = true - } - } - - # module.openai_private_dns_zone.azurerm_private_dns_zone.private_dns_zone will be created - + resource "azurerm_private_dns_zone" "private_dns_zone" { - + id = (known after apply) - + max_number_of_record_sets = (known after apply) - + max_number_of_virtual_network_links = (known after apply) - + max_number_of_virtual_network_links_with_registration = (known after apply) - + name = "privatelink.openai.azure.com" - + number_of_record_sets = (known after apply) - + resource_group_name = (known after apply) - - + soa_record (known after apply) - } - - # module.openai_private_dns_zone.azurerm_private_dns_zone_virtual_network_link.link["AksVNet"] will be created - + resource "azurerm_private_dns_zone_virtual_network_link" "link" { - + id = (known after apply) - + name = "link_to_aksvnet" - + private_dns_zone_name = "privatelink.openai.azure.com" - + registration_enabled = false - + resource_group_name = (known after apply) - + virtual_network_id = (known after apply) - } - - # module.openai_private_endpoint.azurerm_private_endpoint.private_endpoint will be created - + resource "azurerm_private_endpoint" "private_endpoint" { - + custom_dns_configs = (known after apply) - + id = (known after apply) - + location = "westus3" - + name = "OpenAiPrivateEndpoint" - + network_interface = (known after apply) - + private_dns_zone_configs = (known after apply) - + resource_group_name = (known after apply) - + subnet_id = (known after apply) - - + private_dns_zone_group { - + id = (known after apply) - + name = "AcrPrivateDnsZoneGroup" - + private_dns_zone_ids = (known after apply) - } - - + private_service_connection { - + is_manual_connection = false - + name = "OpenAiPrivateEndpointConnection" - + private_connection_resource_id = (known after apply) - + private_ip_address = (known after apply) - + subresource_names = [ - + "account", - ] - } - } - - # module.storage_account.azurerm_storage_account.storage_account will be created - + resource "azurerm_storage_account" "storage_account" { - + access_tier = (known after apply) - + account_kind = "StorageV2" - + account_replication_type = "LRS" - + account_tier = "Standard" - + allow_nested_items_to_be_public = false - + cross_tenant_replication_enabled = false - + default_to_oauth_authentication = false - + dns_endpoint_type = "Standard" - + https_traffic_only_enabled = true - + id = (known after apply) - + infrastructure_encryption_enabled = false - + is_hns_enabled = false - + large_file_share_enabled = (known after apply) - + local_user_enabled = true - + location = "westus3" - + min_tls_version = "TLS1_2" - + name = (known after apply) - + nfsv3_enabled = false - + primary_access_key = (sensitive value) - + primary_blob_connection_string = (sensitive value) - + primary_blob_endpoint = (known after apply) - + primary_blob_host = (known after apply) - + primary_blob_internet_endpoint = (known after apply) - + primary_blob_internet_host = (known after apply) - + primary_blob_microsoft_endpoint = (known after apply) - + primary_blob_microsoft_host = (known after apply) - + primary_connection_string = (sensitive value) - + primary_dfs_endpoint = (known after apply) - + primary_dfs_host = (known after apply) - + primary_dfs_internet_endpoint = (known after apply) - + primary_dfs_internet_host = (known after apply) - + primary_dfs_microsoft_endpoint = (known after apply) - + primary_dfs_microsoft_host = (known after apply) - + primary_file_endpoint = (known after apply) - + primary_file_host = (known after apply) - + primary_file_internet_endpoint = (known after apply) - + primary_file_internet_host = (known after apply) - + primary_file_microsoft_endpoint = (known after apply) - + primary_file_microsoft_host = (known after apply) - + primary_location = (known after apply) - + primary_queue_endpoint = (known after apply) - + primary_queue_host = (known after apply) - + primary_queue_microsoft_endpoint = (known after apply) - + primary_queue_microsoft_host = (known after apply) - + primary_table_endpoint = (known after apply) - + primary_table_host = (known after apply) - + primary_table_microsoft_endpoint = (known after apply) - + primary_table_microsoft_host = (known after apply) - + primary_web_endpoint = (known after apply) - + primary_web_host = (known after apply) - + primary_web_internet_endpoint = (known after apply) - + primary_web_internet_host = (known after apply) - + primary_web_microsoft_endpoint = (known after apply) - + primary_web_microsoft_host = (known after apply) - + public_network_access_enabled = true - + queue_encryption_key_type = "Service" - + resource_group_name = (known after apply) - + secondary_access_key = (sensitive value) - + secondary_blob_connection_string = (sensitive value) - + secondary_blob_endpoint = (known after apply) - + secondary_blob_host = (known after apply) - + secondary_blob_internet_endpoint = (known after apply) - + secondary_blob_internet_host = (known after apply) - + secondary_blob_microsoft_endpoint = (known after apply) - + secondary_blob_microsoft_host = (known after apply) - + secondary_connection_string = (sensitive value) - + secondary_dfs_endpoint = (known after apply) - + secondary_dfs_host = (known after apply) - + secondary_dfs_internet_endpoint = (known after apply) - + secondary_dfs_internet_host = (known after apply) - + secondary_dfs_microsoft_endpoint = (known after apply) - + secondary_dfs_microsoft_host = (known after apply) - + secondary_file_endpoint = (known after apply) - + secondary_file_host = (known after apply) - + secondary_file_internet_endpoint = (known after apply) - + secondary_file_internet_host = (known after apply) - + secondary_file_microsoft_endpoint = (known after apply) - + secondary_file_microsoft_host = (known after apply) - + secondary_location = (known after apply) - + secondary_queue_endpoint = (known after apply) - + secondary_queue_host = (known after apply) - + secondary_queue_microsoft_endpoint = (known after apply) - + secondary_queue_microsoft_host = (known after apply) - + secondary_table_endpoint = (known after apply) - + secondary_table_host = (known after apply) - + secondary_table_microsoft_endpoint = (known after apply) - + secondary_table_microsoft_host = (known after apply) - + secondary_web_endpoint = (known after apply) - + secondary_web_host = (known after apply) - + secondary_web_internet_endpoint = (known after apply) - + secondary_web_internet_host = (known after apply) - + secondary_web_microsoft_endpoint = (known after apply) - + secondary_web_microsoft_host = (known after apply) - + sftp_enabled = false - + shared_access_key_enabled = true - + table_encryption_key_type = "Service" - - + blob_properties (known after apply) - - + identity { - + principal_id = (known after apply) - + tenant_id = (known after apply) - + type = "SystemAssigned" - } - - + network_rules (known after apply) - - + queue_properties (known after apply) - - + routing (known after apply) - - + share_properties (known after apply) - - + static_website (known after apply) - } - - # module.virtual_network.azurerm_monitor_diagnostic_setting.settings will be created - + resource "azurerm_monitor_diagnostic_setting" "settings" { - + id = (known after apply) - + log_analytics_destination_type = (known after apply) - + log_analytics_workspace_id = (known after apply) - + name = "VirtualNetworkDiagnosticsSettings" - + target_resource_id = (known after apply) - - + metric { - + category = "AllMetrics" - + enabled = true - } - } - - # module.virtual_network.azurerm_subnet.subnet["AzureBastionSubnet"] will be created - + resource "azurerm_subnet" "subnet" { - + address_prefixes = [ - + "10.243.2.0/24", - ] - + default_outbound_access_enabled = true - + id = (known after apply) - + name = "AzureBastionSubnet" - + private_endpoint_network_policies = "Enabled" - + private_link_service_network_policies_enabled = false - + resource_group_name = (known after apply) - + virtual_network_name = "AksVNet" - } - - # module.virtual_network.azurerm_subnet.subnet["PodSubnet"] will be created - + resource "azurerm_subnet" "subnet" { - + address_prefixes = [ - + "10.242.0.0/16", - ] - + default_outbound_access_enabled = true - + id = (known after apply) - + name = "PodSubnet" - + private_endpoint_network_policies = "Enabled" - + private_link_service_network_policies_enabled = false - + resource_group_name = (known after apply) - + virtual_network_name = "AksVNet" - - + delegation { - + name = "delegation" - - + service_delegation { - + actions = [ - + "Microsoft.Network/virtualNetworks/subnets/join/action", - ] - + name = "Microsoft.ContainerService/managedClusters" - } - } - } - - # module.virtual_network.azurerm_subnet.subnet["SystemSubnet"] will be created - + resource "azurerm_subnet" "subnet" { - + address_prefixes = [ - + "10.240.0.0/16", - ] - + default_outbound_access_enabled = true - + id = (known after apply) - + name = "SystemSubnet" - + private_endpoint_network_policies = "Enabled" - + private_link_service_network_policies_enabled = false - + resource_group_name = (known after apply) - + virtual_network_name = "AksVNet" - } - - # module.virtual_network.azurerm_subnet.subnet["UserSubnet"] will be created - + resource "azurerm_subnet" "subnet" { - + address_prefixes = [ - + "10.241.0.0/16", - ] - + default_outbound_access_enabled = true - + id = (known after apply) - + name = "UserSubnet" - + private_endpoint_network_policies = "Enabled" - + private_link_service_network_policies_enabled = false - + resource_group_name = (known after apply) - + virtual_network_name = "AksVNet" - } - - # module.virtual_network.azurerm_subnet.subnet["VmSubnet"] will be created - + resource "azurerm_subnet" "subnet" { - + address_prefixes = [ - + "10.243.1.0/24", - ] - + default_outbound_access_enabled = true - + id = (known after apply) - + name = "VmSubnet" - + private_endpoint_network_policies = "Enabled" - + private_link_service_network_policies_enabled = false - + resource_group_name = (known after apply) - + virtual_network_name = "AksVNet" - } - - # module.virtual_network.azurerm_virtual_network.vnet will be created - + resource "azurerm_virtual_network" "vnet" { - + address_space = [ - + "10.0.0.0/8", - ] - + dns_servers = (known after apply) - + guid = (known after apply) - + id = (known after apply) - + location = "westus3" - + name = "AksVNet" - + private_endpoint_vnet_policies = "Disabled" - + resource_group_name = (known after apply) - + subnet = (known after apply) - } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/run.sh b/scenarios/AksOpenAiTerraform/run.sh deleted file mode 100644 index b25a2012f..000000000 --- a/scenarios/AksOpenAiTerraform/run.sh +++ /dev/null @@ -1,99 +0,0 @@ -export RG_NAME="" - -export OPEN_AI_SUBDOMAIN="magic8ball" - -# Publish Image -export ACR_NAME=(terraform output -raw acr_name) -export IMAGE="azurecr.io/magic8ball:latest" - -# Nginx Ingress Controller -export nginxNamespace="ingress-basic" -export nginxRepoName="ingress-nginx" -export nginxRepoUrl="https://kubernetes.github.io/ingress-nginx" -export nginxChartName="ingress-nginx" -export nginxReleaseName="nginx-ingress" -export nginxReplicaCount=3 - -# Certificate Manager -export cmNamespace="cert-manager" -export cmRepoName="jetstack" -export cmRepoUrl="https://charts.jetstack.io" -export cmChartName="cert-manager" -export cmReleaseName="cert-manager" - -# Cluster Issuer -email="paolos@microsoft.com" -clusterIssuerName="letsencrypt-nginx" -clusterIssuerTemplate="cluster-issuer.yml" - -# Variables -acrName="CyanAcr" -acrResourceGrougName="CyanRG" -location="FranceCentral" -attachAcr=false -imageName="magic8ball" -tag="v2" -containerName="magic8ball" -image="$acrName.azurecr.io/$imageName:$tag" -imagePullPolicy="IfNotPresent" # Always, Never, IfNotPresent -managedIdentityName="CyanWorkloadManagedIdentity" -federatedIdentityName="Magic8BallFederatedIdentity" - -# Azure Subscription and Tenant -subscriptionId=$(az account show --query id --output tsv) -subscriptionName=$(az account show --query name --output tsv) -tenantId=$(az account show --query tenantId --output tsv) - -# Parameters -title="Magic 8 Ball" -label="Pose your question and cross your fingers!" -temperature="0.9" -imageWidth="80" - -# OpenAI -openAiName="CyanOpenAi " -openAiResourceGroupName="CyanRG" -openAiType="azure_ad" -openAiBase="https://cyanopenai.openai.azure.com/" -openAiModel="gpt-35-turbo" -openAiDeployment="gpt-35-turbo" - -# Nginx Ingress Controller -nginxNamespace="ingress-basic" -nginxRepoName="ingress-nginx" -nginxRepoUrl="https://kubernetes.github.io/ingress-nginx" -nginxChartName="ingress-nginx" -nginxReleaseName="nginx-ingress" -nginxReplicaCount=3 - -# Certificate Manager -cmNamespace="cert-manager" -cmRepoName="jetstack" -cmRepoUrl="https://charts.jetstack.io" -cmChartName="cert-manager" -cmReleaseName="cert-manager" - -# Cluster Issuer -email="paolos@microsoft.com" -clusterIssuerName="letsencrypt-nginx" -clusterIssuerTemplate="cluster-issuer.yml" - -# AKS Cluster -aksClusterName="CyanAks" -aksResourceGroupName="CyanRG" - -# Sample Application -namespace="magic8ball" -serviceAccountName="magic8ball-sa" -deploymentTemplate="deployment.yml" -serviceTemplate="service.yml" -configMapTemplate="configMap.yml" -secretTemplate="secret.yml" - -# Ingress and DNS -ingressTemplate="ingress.yml" -ingressName="magic8ball-ingress" -dnsZoneName="contoso.com" -dnsZoneResourceGroupName="DnsResourceGroup" -subdomain="magic" -host="$subdomain.$dnsZoneName" \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/.env b/scenarios/AksOpenAiTerraform/scripts/.env deleted file mode 100644 index 9af98b868..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/.env +++ /dev/null @@ -1,2 +0,0 @@ -AZURE_OPENAI_TYPE="azure_ad" -AZURE_OPENAI_BASE="https://myopenai.openai.azure.com/" \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/00-variables.sh b/scenarios/AksOpenAiTerraform/scripts/00-variables.sh deleted file mode 100644 index 38abccfb6..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/00-variables.sh +++ /dev/null @@ -1,71 +0,0 @@ -# Variables -acrName="CyanAcr" -acrResourceGrougName="CyanRG" -location="FranceCentral" -attachAcr=false -imageName="magic8ball" -tag="v2" -containerName="magic8ball" -image="$acrName.azurecr.io/$imageName:$tag" -imagePullPolicy="IfNotPresent" # Always, Never, IfNotPresent -managedIdentityName="CyanWorkloadManagedIdentity" -federatedIdentityName="Magic8BallFederatedIdentity" - -# Azure Subscription and Tenant -subscriptionId=$(az account show --query id --output tsv) -subscriptionName=$(az account show --query name --output tsv) -tenantId=$(az account show --query tenantId --output tsv) - -# Parameters -title="Magic 8 Ball" -label="Pose your question and cross your fingers!" -temperature="0.9" -imageWidth="80" - -# OpenAI -openAiName="CyanOpenAi " -openAiResourceGroupName="CyanRG" -openAiType="azure_ad" -openAiBase="https://cyanopenai.openai.azure.com/" -openAiModel="gpt-35-turbo" -openAiDeployment="gpt-35-turbo" - -# Nginx Ingress Controller -nginxNamespace="ingress-basic" -nginxRepoName="ingress-nginx" -nginxRepoUrl="https://kubernetes.github.io/ingress-nginx" -nginxChartName="ingress-nginx" -nginxReleaseName="nginx-ingress" -nginxReplicaCount=3 - -# Certificate Manager -cmNamespace="cert-manager" -cmRepoName="jetstack" -cmRepoUrl="https://charts.jetstack.io" -cmChartName="cert-manager" -cmReleaseName="cert-manager" - -# Cluster Issuer -email="paolos@microsoft.com" -clusterIssuerName="letsencrypt-nginx" -clusterIssuerTemplate="cluster-issuer.yml" - -# AKS Cluster -aksClusterName="CyanAks" -aksResourceGroupName="CyanRG" - -# Sample Application -namespace="magic8ball" -serviceAccountName="magic8ball-sa" -deploymentTemplate="deployment.yml" -serviceTemplate="service.yml" -configMapTemplate="configMap.yml" -secretTemplate="secret.yml" - -# Ingress and DNS -ingressTemplate="ingress.yml" -ingressName="magic8ball-ingress" -dnsZoneName="contoso.com" -dnsZoneResourceGroupName="DnsResourceGroup" -subdomain="magic" -host="$subdomain.$dnsZoneName" \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/01-build-docker-image.sh b/scenarios/AksOpenAiTerraform/scripts/01-build-docker-image.sh deleted file mode 100644 index 1425afefb..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/01-build-docker-image.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# For more information, see: -# * https://hub.docker.com/_/python -# * https://docs.streamlit.io/knowledge-base/tutorials/deploy/docker -# * https://stackoverflow.com/questions/30494050/how-do-i-pass-environment-variables-to-docker-containers - -# Variables -source ./00-variables.sh - -# Build the docker image -docker build -t $imageName:$tag -f Dockerfile . \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/wip/01-push-app-image.sh b/scenarios/AksOpenAiTerraform/scripts/01-push-app-image.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/wip/01-push-app-image.sh rename to scenarios/AksOpenAiTerraform/scripts/01-push-app-image.sh diff --git a/scenarios/AksOpenAiTerraform/scripts/02-run-docker-container.sh b/scenarios/AksOpenAiTerraform/scripts/02-run-docker-container.sh deleted file mode 100644 index 31e4d7f49..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/02-run-docker-container.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# For more information, see: -# * https://hub.docker.com/_/python -# * https://docs.streamlit.io/knowledge-base/tutorials/deploy/docker -# * https://stackoverflow.com/questions/30494050/how-do-i-pass-environment-variables-to-docker-containers - -# Variables -source ./00-variables.sh - -# Run the docker container -docker run -it \ - --rm \ - -p 8501:8501 \ - -e TEMPERATURE=$temperature \ - -e AZURE_OPENAI_BASE=$AZURE_OPENAI_BASE \ - -e AZURE_OPENAI_KEY=$AZURE_OPENAI_KEY \ - -e AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL \ - -e AZURE_OPENAI_DEPLOYMENT=$AZURE_OPENAI_DEPLOYMENT \ - --name $containerName \ - $imageName:$tag \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/03-push-docker-image.sh b/scenarios/AksOpenAiTerraform/scripts/03-push-docker-image.sh deleted file mode 100644 index e0e9865a9..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/03-push-docker-image.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Variables -source ./00-variables.sh - -# Login to ACR -az acr login --name $acrName - -# Retrieve ACR login server. Each container image needs to be tagged with the loginServer name of the registry. -loginServer=$(az acr show --name $acrName --query loginServer --output tsv) - -# Tag the local image with the loginServer of ACR -docker tag ${imageName,,}:$tag $loginServer/${imageName,,}:$tag - -# Push latest container image to ACR -docker push $loginServer/${imageName,,}:$tag \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh b/scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh index 4e2670847..f059c37ea 100644 --- a/scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh +++ b/scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh @@ -1,8 +1,5 @@ #!/bin/bash -# Variables -source ./00-variables.sh - # Use Helm to deploy an NGINX ingress controller result=$(helm list -n $nginxNamespace | grep $nginxReleaseName | awk '{print $1}') diff --git a/scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh b/scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh index 590a41436..3fee03e52 100644 --- a/scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh +++ b/scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh @@ -1,8 +1,5 @@ #/bin/bash -# Variables -source ./00-variables.sh - # Check if the ingress-nginx repository is not already added result=$(helm repo list | grep $cmRepoName | awk '{print $1}') diff --git a/scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh b/scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh index fd7976cfb..9ab805a54 100644 --- a/scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh +++ b/scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh @@ -1,8 +1,5 @@ #/bin/bash -# Variables -source ./00-variables.sh - # Check if the cluster issuer already exists result=$(kubectl get ClusterIssuer -o json | jq -r '.items[].metadata.name | select(. == "'$clusterIssuerName'")') diff --git a/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh b/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh index 3843f71b7..f9e1d757c 100644 --- a/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh +++ b/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh @@ -3,15 +3,6 @@ # Variables source ./00-variables.sh -# Attach ACR to AKS cluster -if [[ $attachAcr == true ]]; then - echo "Attaching ACR $acrName to AKS cluster $aksClusterName..." - az aks update \ - --name $aksClusterName \ - --resource-group $aksResourceGroupName \ - --attach-acr $acrName -fi - # Check if namespace exists in the cluster result=$(kubectl get namespace -o jsonpath="{.items[?(@.metadata.name=='$namespace')].metadata.name}") diff --git a/scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh b/scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh index 388518355..52f090706 100644 --- a/scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh +++ b/scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh @@ -1,8 +1,5 @@ #/bin/bash -# Variables -source ./00-variables.sh - # Create the ingress echo "[$ingressName] ingress does not exist" echo "Creating [$ingressName] ingress..." diff --git a/scenarios/AksOpenAiTerraform/scripts/Dockerfile b/scenarios/AksOpenAiTerraform/scripts/app/Dockerfile similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/Dockerfile rename to scenarios/AksOpenAiTerraform/scripts/app/Dockerfile diff --git a/scenarios/AksOpenAiTerraform/scripts/app.py b/scenarios/AksOpenAiTerraform/scripts/app/app.py similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/app.py rename to scenarios/AksOpenAiTerraform/scripts/app/app.py diff --git a/scenarios/AksOpenAiTerraform/scripts/images/magic8ball.png b/scenarios/AksOpenAiTerraform/scripts/app/images/magic8ball.png similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/images/magic8ball.png rename to scenarios/AksOpenAiTerraform/scripts/app/images/magic8ball.png diff --git a/scenarios/AksOpenAiTerraform/scripts/images/robot.png b/scenarios/AksOpenAiTerraform/scripts/app/images/robot.png similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/images/robot.png rename to scenarios/AksOpenAiTerraform/scripts/app/images/robot.png diff --git a/scenarios/AksOpenAiTerraform/scripts/requirements.txt b/scenarios/AksOpenAiTerraform/scripts/app/requirements.txt similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/requirements.txt rename to scenarios/AksOpenAiTerraform/scripts/app/requirements.txt diff --git a/scenarios/AksOpenAiTerraform/wip/install-nginx-via-helm-and-create-sa.sh b/scenarios/AksOpenAiTerraform/scripts/install-nginx-via-helm-and-create-sa.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/wip/install-nginx-via-helm-and-create-sa.sh rename to scenarios/AksOpenAiTerraform/scripts/install-nginx-via-helm-and-create-sa.sh diff --git a/scenarios/AksOpenAiTerraform/scripts/cluster-issuer.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/cluster-issuer.yml rename to scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml diff --git a/scenarios/AksOpenAiTerraform/scripts/configMap.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/configMap.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/configMap.yml rename to scenarios/AksOpenAiTerraform/scripts/manifests/configMap.yml diff --git a/scenarios/AksOpenAiTerraform/scripts/deployment.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/deployment.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/deployment.yml rename to scenarios/AksOpenAiTerraform/scripts/manifests/deployment.yml diff --git a/scenarios/AksOpenAiTerraform/scripts/ingress.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/ingress.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/ingress.yml rename to scenarios/AksOpenAiTerraform/scripts/manifests/ingress.yml diff --git a/scenarios/AksOpenAiTerraform/scripts/service.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/service.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/service.yml rename to scenarios/AksOpenAiTerraform/scripts/manifests/service.yml diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index ca6f361de..9bc0a2840 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -10,7 +10,7 @@ variable "location" { variable "openai_subdomain" { type = string - default = "magic8ball" + default = "magic8ball-test465544" } # -test465544 diff --git a/scenarios/AksOpenAiTerraform/wip/04-create-nginx-ingress-controller.sh b/scenarios/AksOpenAiTerraform/wip/04-create-nginx-ingress-controller.sh deleted file mode 100644 index f059c37ea..000000000 --- a/scenarios/AksOpenAiTerraform/wip/04-create-nginx-ingress-controller.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Use Helm to deploy an NGINX ingress controller -result=$(helm list -n $nginxNamespace | grep $nginxReleaseName | awk '{print $1}') - -if [[ -n $result ]]; then - echo "[$nginxReleaseName] ingress controller already exists in the [$nginxNamespace] namespace" -else - # Check if the ingress-nginx repository is not already added - result=$(helm repo list | grep $nginxRepoName | awk '{print $1}') - - if [[ -n $result ]]; then - echo "[$nginxRepoName] Helm repo already exists" - else - # Add the ingress-nginx repository - echo "Adding [$nginxRepoName] Helm repo..." - helm repo add $nginxRepoName $nginxRepoUrl - fi - - # Update your local Helm chart repository cache - echo 'Updating Helm repos...' - helm repo update - - # Deploy NGINX ingress controller - echo "Deploying [$nginxReleaseName] NGINX ingress controller to the [$nginxNamespace] namespace..." - helm install $nginxReleaseName $nginxRepoName/$nginxChartName \ - --create-namespace \ - --namespace $nginxNamespace \ - --set controller.nodeSelector."kubernetes\.io/os"=linux \ - --set controller.replicaCount=$replicaCount \ - --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ - --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz -fi - -# Get values -helm get values $nginxReleaseName --namespace $nginxNamespace diff --git a/scenarios/AksOpenAiTerraform/wip/05-install-cert-manager.sh b/scenarios/AksOpenAiTerraform/wip/05-install-cert-manager.sh deleted file mode 100644 index 3fee03e52..000000000 --- a/scenarios/AksOpenAiTerraform/wip/05-install-cert-manager.sh +++ /dev/null @@ -1,31 +0,0 @@ -#/bin/bash - -# Check if the ingress-nginx repository is not already added -result=$(helm repo list | grep $cmRepoName | awk '{print $1}') - -if [[ -n $result ]]; then - echo "[$cmRepoName] Helm repo already exists" -else - # Add the Jetstack Helm repository - echo "Adding [$cmRepoName] Helm repo..." - helm repo add $cmRepoName $cmRepoUrl -fi - -# Update your local Helm chart repository cache -echo 'Updating Helm repos...' -helm repo update - -# Install cert-manager Helm chart -result=$(helm list -n $cmNamespace | grep $cmReleaseName | awk '{print $1}') - -if [[ -n $result ]]; then - echo "[$cmReleaseName] cert-manager already exists in the $cmNamespace namespace" -else - # Install the cert-manager Helm chart - echo "Deploying [$cmReleaseName] cert-manager to the $cmNamespace namespace..." - helm install $cmReleaseName $cmRepoName/$cmChartName \ - --create-namespace \ - --namespace $cmNamespace \ - --set installCRDs=true \ - --set nodeSelector."kubernetes\.io/os"=linux -fi diff --git a/scenarios/AksOpenAiTerraform/wip/06-create-cluster-issuer.sh b/scenarios/AksOpenAiTerraform/wip/06-create-cluster-issuer.sh deleted file mode 100644 index 9ab805a54..000000000 --- a/scenarios/AksOpenAiTerraform/wip/06-create-cluster-issuer.sh +++ /dev/null @@ -1,16 +0,0 @@ -#/bin/bash - -# Check if the cluster issuer already exists -result=$(kubectl get ClusterIssuer -o json | jq -r '.items[].metadata.name | select(. == "'$clusterIssuerName'")') - -if [[ -n $result ]]; then - echo "[$clusterIssuerName] cluster issuer already exists" - exit -else - # Create the cluster issuer - echo "[$clusterIssuerName] cluster issuer does not exist" - echo "Creating [$clusterIssuerName] cluster issuer..." - cat $clusterIssuerTemplate | - yq "(.spec.acme.email)|="\""$email"\" | - kubectl apply -f - -fi diff --git a/scenarios/AksOpenAiTerraform/wip/07-create-workload-managed-identity.sh b/scenarios/AksOpenAiTerraform/wip/07-create-workload-managed-identity.sh deleted file mode 100644 index c770e6476..000000000 --- a/scenarios/AksOpenAiTerraform/wip/07-create-workload-managed-identity.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash - -# Variables -source ./00-variables.sh - -# Check if the user-assigned managed identity already exists -echo "Checking if [$managedIdentityName] user-assigned managed identity actually exists in the [$aksResourceGroupName] resource group..." - -az identity show \ - --name $managedIdentityName \ - --resource-group $aksResourceGroupName &>/dev/null - -if [[ $? != 0 ]]; then - echo "No [$managedIdentityName] user-assigned managed identity actually exists in the [$aksResourceGroupName] resource group" - echo "Creating [$managedIdentityName] user-assigned managed identity in the [$aksResourceGroupName] resource group..." - - # Create the user-assigned managed identity - az identity create \ - --name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --location $location \ - --subscription $subscriptionId 1>/dev/null - - if [[ $? == 0 ]]; then - echo "[$managedIdentityName] user-assigned managed identity successfully created in the [$aksResourceGroupName] resource group" - else - echo "Failed to create [$managedIdentityName] user-assigned managed identity in the [$aksResourceGroupName] resource group" - exit - fi -else - echo "[$managedIdentityName] user-assigned managed identity already exists in the [$aksResourceGroupName] resource group" -fi - -# Retrieve the clientId of the user-assigned managed identity -echo "Retrieving clientId for [$managedIdentityName] managed identity..." -clientId=$(az identity show \ - --name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --query clientId \ - --output tsv) - -if [[ -n $clientId ]]; then - echo "[$clientId] clientId for the [$managedIdentityName] managed identity successfully retrieved" -else - echo "Failed to retrieve clientId for the [$managedIdentityName] managed identity" - exit -fi - -# Retrieve the principalId of the user-assigned managed identity -echo "Retrieving principalId for [$managedIdentityName] managed identity..." -principalId=$(az identity show \ - --name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --query principalId \ - --output tsv) - -if [[ -n $principalId ]]; then - echo "[$principalId] principalId for the [$managedIdentityName] managed identity successfully retrieved" -else - echo "Failed to retrieve principalId for the [$managedIdentityName] managed identity" - exit -fi - -# Get the resource id of the Azure OpenAI resource -openAiId=$(az cognitiveservices account show \ - --name $openAiName \ - --resource-group $openAiResourceGroupName \ - --query id \ - --output tsv) - -if [[ -n $openAiId ]]; then - echo "Resource id for the [$openAiName] Azure OpenAI resource successfully retrieved" -else - echo "Failed to the resource id for the [$openAiName] Azure OpenAI resource" - exit -1 -fi - -# Assign the Cognitive Services User role on the Azure OpenAI resource to the managed identity -role="Cognitive Services User" -echo "Checking if the [$managedIdentityName] managed identity has been assigned to [$role] role with [$openAiName] Azure OpenAI resource as a scope..." -current=$(az role assignment list \ - --assignee $principalId \ - --scope $openAiId \ - --query "[?roleDefinitionName=='$role'].roleDefinitionName" \ - --output tsv 2>/dev/null) - -if [[ $current == $role ]]; then - echo "[$managedIdentityName] managed identity is already assigned to the ["$current"] role with [$openAiName] Azure OpenAI resource as a scope" -else - echo "[$managedIdentityName] managed identity is not assigned to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" - echo "Assigning the [$role] role to the [$managedIdentityName] managed identity with [$openAiName] Azure OpenAI resource as a scope..." - - az role assignment create \ - --assignee $principalId \ - --role "$role" \ - --scope $openAiId 1>/dev/null - - if [[ $? == 0 ]]; then - echo "[$managedIdentityName] managed identity successfully assigned to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" - else - echo "Failed to assign the [$managedIdentityName] managed identity to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" - exit - fi -fi \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/wip/08-create-service-account.sh b/scenarios/AksOpenAiTerraform/wip/08-create-service-account.sh deleted file mode 100644 index 5a89a0619..000000000 --- a/scenarios/AksOpenAiTerraform/wip/08-create-service-account.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -# Variables for the user-assigned managed identity -source ./00-variables.sh - -# Check if the namespace already exists -result=$(kubectl get namespace -o 'jsonpath={.items[?(@.metadata.name=="'$namespace'")].metadata.name'}) - -if [[ -n $result ]]; then - echo "[$namespace] namespace already exists" -else - # Create the namespace for your ingress resources - echo "[$namespace] namespace does not exist" - echo "Creating [$namespace] namespace..." - kubectl create namespace $namespace -fi - -# Check if the service account already exists -result=$(kubectl get sa -n $namespace -o 'jsonpath={.items[?(@.metadata.name=="'$serviceAccountName'")].metadata.name'}) - -if [[ -n $result ]]; then - echo "[$serviceAccountName] service account already exists" -else - # Retrieve the resource id of the user-assigned managed identity - echo "Retrieving clientId for [$managedIdentityName] managed identity..." - managedIdentityClientId=$(az identity show \ - --name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --query clientId \ - --output tsv) - - if [[ -n $managedIdentityClientId ]]; then - echo "[$managedIdentityClientId] clientId for the [$managedIdentityName] managed identity successfully retrieved" - else - echo "Failed to retrieve clientId for the [$managedIdentityName] managed identity" - exit - fi - - # Create the service account - echo "[$serviceAccountName] service account does not exist" - echo "Creating [$serviceAccountName] service account..." - cat </dev/null - -if [[ $? != 0 ]]; then - echo "No [$federatedIdentityName] federated identity credential actually exists in the [$aksResourceGroupName] resource group" - - # Get the OIDC Issuer URL - aksOidcIssuerUrl="$(az aks show \ - --only-show-errors \ - --name $aksClusterName \ - --resource-group $aksResourceGroupName \ - --query oidcIssuerProfile.issuerUrl \ - --output tsv)" - - # Show OIDC Issuer URL - if [[ -n $aksOidcIssuerUrl ]]; then - echo "The OIDC Issuer URL of the $aksClusterName cluster is $aksOidcIssuerUrl" - fi - - echo "Creating [$federatedIdentityName] federated identity credential in the [$aksResourceGroupName] resource group..." - - # Establish the federated identity credential between the managed identity, the service account issuer, and the subject. - az identity federated-credential create \ - --name $federatedIdentityName \ - --identity-name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --issuer $aksOidcIssuerUrl \ - --subject system:serviceaccount:$namespace:$serviceAccountName - - if [[ $? == 0 ]]; then - echo "[$federatedIdentityName] federated identity credential successfully created in the [$aksResourceGroupName] resource group" - else - echo "Failed to create [$federatedIdentityName] federated identity credential in the [$aksResourceGroupName] resource group" - exit - fi -else - echo "[$federatedIdentityName] federated identity credential already exists in the [$aksResourceGroupName] resource group" -fi \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/wip/09-deploy-app.sh b/scenarios/AksOpenAiTerraform/wip/09-deploy-app.sh deleted file mode 100644 index f9e1d757c..000000000 --- a/scenarios/AksOpenAiTerraform/wip/09-deploy-app.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Variables -source ./00-variables.sh - -# Check if namespace exists in the cluster -result=$(kubectl get namespace -o jsonpath="{.items[?(@.metadata.name=='$namespace')].metadata.name}") - -if [[ -n $result ]]; then - echo "$namespace namespace already exists in the cluster" -else - echo "$namespace namespace does not exist in the cluster" - echo "creating $namespace namespace in the cluster..." - kubectl create namespace $namespace -fi - -# Create config map -cat $configMapTemplate | - yq "(.data.TITLE)|="\""$title"\" | - yq "(.data.LABEL)|="\""$label"\" | - yq "(.data.TEMPERATURE)|="\""$temperature"\" | - yq "(.data.IMAGE_WIDTH)|="\""$imageWidth"\" | - yq "(.data.AZURE_OPENAI_TYPE)|="\""$openAiType"\" | - yq "(.data.AZURE_OPENAI_BASE)|="\""$openAiBase"\" | - yq "(.data.AZURE_OPENAI_MODEL)|="\""$openAiModel"\" | - yq "(.data.AZURE_OPENAI_DEPLOYMENT)|="\""$openAiDeployment"\" | - kubectl apply -n $namespace -f - - -# Create deployment -cat $deploymentTemplate | - yq "(.spec.template.spec.containers[0].image)|="\""$image"\" | - yq "(.spec.template.spec.containers[0].imagePullPolicy)|="\""$imagePullPolicy"\" | - yq "(.spec.template.spec.serviceAccountName)|="\""$serviceAccountName"\" | - kubectl apply -n $namespace -f - - -# Create deployment -kubectl apply -f $serviceTemplate -n $namespace \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/wip/10-create-ingress.sh b/scenarios/AksOpenAiTerraform/wip/10-create-ingress.sh deleted file mode 100644 index 52f090706..000000000 --- a/scenarios/AksOpenAiTerraform/wip/10-create-ingress.sh +++ /dev/null @@ -1,9 +0,0 @@ -#/bin/bash - -# Create the ingress -echo "[$ingressName] ingress does not exist" -echo "Creating [$ingressName] ingress..." -cat $ingressTemplate | - yq "(.spec.tls[0].hosts[0])|="\""$host"\" | - yq "(.spec.rules[0].host)|="\""$host"\" | - kubectl apply -n $namespace -f - \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/wip/11-configure-dns.sh b/scenarios/AksOpenAiTerraform/wip/11-configure-dns.sh deleted file mode 100644 index 95f8baf69..000000000 --- a/scenarios/AksOpenAiTerraform/wip/11-configure-dns.sh +++ /dev/null @@ -1,79 +0,0 @@ -# Variables -source ./00-variables.sh - -# Retrieve the public IP address from the ingress -echo "Retrieving the external IP address from the [$ingressName] ingress..." -publicIpAddress=$(kubectl get ingress $ingressName -n $namespace -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - -if [ -n $publicIpAddress ]; then - echo "[$publicIpAddress] external IP address of the application gateway ingress controller successfully retrieved from the [$ingressName] ingress" -else - echo "Failed to retrieve the external IP address of the application gateway ingress controller from the [$ingressName] ingress" - exit -fi - -# Check if an A record for todolist subdomain exists in the DNS Zone -echo "Retrieving the A record for the [$subdomain] subdomain from the [$dnsZoneName] DNS zone..." -ipv4Address=$(az network dns record-set a list \ - --zone-name $dnsZoneName \ - --resource-group $dnsZoneResourceGroupName \ - --query "[?name=='$subdomain'].arecords[].ipv4Address" \ - --output tsv) - -if [[ -n $ipv4Address ]]; then - echo "An A record already exists in [$dnsZoneName] DNS zone for the [$subdomain] subdomain with [$ipv4Address] IP address" - - if [[ $ipv4Address == $publicIpAddress ]]; then - echo "The [$ipv4Address] ip address of the existing A record is equal to the ip address of the [$ingressName] ingress" - echo "No additional step is required" - exit - else - echo "The [$ipv4Address] ip address of the existing A record is different than the ip address of the [$ingressName] ingress" - fi - - # Retrieving name of the record set relative to the zone - echo "Retrieving the name of the record set relative to the [$dnsZoneName] zone..." - - recordSetName=$(az network dns record-set a list \ - --zone-name $dnsZoneName \ - --resource-group $dnsZoneResourceGroupName \ - --query "[?name=='$subdomain'].name" \ - --output name 2>/dev/null) - - if [[ -n $recordSetName ]]; then - "[$recordSetName] record set name successfully retrieved" - else - "Failed to retrieve the name of the record set relative to the [$dnsZoneName] zone" - exit - fi - - # Remove the a record - echo "Removing the A record from the record set relative to the [$dnsZoneName] zone..." - - az network dns record-set a remove-record \ - --ipv4-address $ipv4Address \ - --record-set-name $recordSetName \ - --zone-name $dnsZoneName \ - --resource-group $dnsZoneResourceGroupName - - if [[ $? == 0 ]]; then - echo "[$ipv4Address] ip address successfully removed from the [$recordSetName] record set" - else - echo "Failed to remove the [$ipv4Address] ip address from the [$recordSetName] record set" - exit - fi -fi - -# Create the a record -echo "Creating an A record in [$dnsZoneName] DNS zone for the [$subdomain] subdomain with [$publicIpAddress] IP address..." -az network dns record-set a add-record \ - --zone-name $dnsZoneName \ - --resource-group $dnsZoneResourceGroupName \ - --record-set-name $subdomain \ - --ipv4-address $publicIpAddress 1>/dev/null - -if [[ $? == 0 ]]; then - echo "A record for the [$subdomain] subdomain with [$publicIpAddress] IP address successfully created in [$dnsZoneName] DNS zone" -else - echo "Failed to create an A record for the $subdomain subdomain with [$publicIpAddress] IP address in [$dnsZoneName] DNS zone" -fi diff --git a/scenarios/AksOpenAiTerraform/wip/app/Dockerfile b/scenarios/AksOpenAiTerraform/wip/app/Dockerfile deleted file mode 100644 index 2f603014f..000000000 --- a/scenarios/AksOpenAiTerraform/wip/app/Dockerfile +++ /dev/null @@ -1,94 +0,0 @@ -# app/Dockerfile - -# # Stage 1 - Install build dependencies - -# A Dockerfile must start with a FROM instruction which sets the base image for the container. -# The Python images come in many flavors, each designed for a specific use case. -# The python:3.11-slim image is a good base image for most applications. -# It is a minimal image built on top of Debian Linux and includes only the necessary packages to run Python. -# The slim image is a good choice because it is small and contains only the packages needed to run Python. -# For more information, see: -# * https://hub.docker.com/_/python -# * https://docs.streamlit.io/knowledge-base/tutorials/deploy/docker -FROM python:3.11-slim AS builder - -# The WORKDIR instruction sets the working directory for any RUN, CMD, ENTRYPOINT, COPY and ADD instructions that follow it in the Dockerfile. -# If the WORKDIR doesn’t exist, it will be created even if it’s not used in any subsequent Dockerfile instruction. -# For more information, see: https://docs.docker.com/engine/reference/builder/#workdir -WORKDIR /app - -# Set environment variables. -# The ENV instruction sets the environment variable to the value . -# This value will be in the environment of all “descendant” Dockerfile commands and can be replaced inline in many as well. -# For more information, see: https://docs.docker.com/engine/reference/builder/#env -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 - -# Install git so that we can clone the app code from a remote repo using the RUN instruction. -# The RUN comand has 2 forms: -# * RUN (shell form, the command is run in a shell, which by default is /bin/sh -c on Linux or cmd /S /C on Windows) -# * RUN ["executable", "param1", "param2"] (exec form) -# The RUN instruction will execute any commands in a new layer on top of the current image and commit the results. -# The resulting committed image will be used for the next step in the Dockerfile. -# For more information, see: https://docs.docker.com/engine/reference/builder/#run -RUN apt-get update && apt-get install -y \ - build-essential \ - curl \ - software-properties-common \ - git \ - && rm -rf /var/lib/apt/lists/* - -# Create a virtualenv to keep dependencies together -RUN python -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" - -# Clone the requirements.txt which contains dependencies to WORKDIR -# COPY has two forms: -# * COPY (this copies the files from the local machine to the container's own filesystem) -# * COPY ["",... ""] (this form is required for paths containing whitespace) -# For more information, see: https://docs.docker.com/engine/reference/builder/#copy -COPY requirements.txt . - -# Install the Python dependencies -RUN pip install --no-cache-dir --no-deps -r requirements.txt - -# Stage 2 - Copy only necessary files to the runner stage - -# The FROM instruction initializes a new build stage for the application -FROM python:3.11-slim - -# Sets the working directory to /app -WORKDIR /app - -# Copy the virtual environment from the builder stage -COPY --from=builder /opt/venv /opt/venv - -# Set environment variables -ENV PATH="/opt/venv/bin:$PATH" - -# Clone the app.py containing the application code -COPY app.py . - -# Copy the images folder to WORKDIR -# The ADD instruction copies new files, directories or remote file URLs from and adds them to the filesystem of the image at the path . -# For more information, see: https://docs.docker.com/engine/reference/builder/#add -ADD images ./images - -# The EXPOSE instruction informs Docker that the container listens on the specified network ports at runtime. -# For more information, see: https://docs.docker.com/engine/reference/builder/#expose -EXPOSE 8501 - -# The HEALTHCHECK instruction has two forms: -# * HEALTHCHECK [OPTIONS] CMD command (check container health by running a command inside the container) -# * HEALTHCHECK NONE (disable any healthcheck inherited from the base image) -# The HEALTHCHECK instruction tells Docker how to test a container to check that it is still working. -# This can detect cases such as a web server that is stuck in an infinite loop and unable to handle new connections, -# even though the server process is still running. For more information, see: https://docs.docker.com/engine/reference/builder/#healthcheck -HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health - -# The ENTRYPOINT instruction has two forms: -# * ENTRYPOINT ["executable", "param1", "param2"] (exec form, preferred) -# * ENTRYPOINT command param1 param2 (shell form) -# The ENTRYPOINT instruction allows you to configure a container that will run as an executable. -# For more information, see: https://docs.docker.com/engine/reference/builder/#entrypoint -ENTRYPOINT ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"] \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/wip/app/app.py b/scenarios/AksOpenAiTerraform/wip/app/app.py deleted file mode 100644 index 4211c57ca..000000000 --- a/scenarios/AksOpenAiTerraform/wip/app/app.py +++ /dev/null @@ -1,347 +0,0 @@ -""" -MIT License - -Copyright (c) 2023 Paolo Salvatori - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -# This sample is based on the following article: -# -# - https://levelup.gitconnected.com/its-time-to-create-a-private-chatgpt-for-yourself-today-6503649e7bb6 -# -# Use pip to install the following packages: -# -# - streamlit -# - openai -# - streamlit-chat -# - azure.identity -# - dotenv -# -# Make sure to provide a value for the following environment variables: -# -# - AZURE_OPENAI_BASE: the URL of your Azure OpenAI resource, for example https://eastus.api.cognitive.microsoft.com/ -# - AZURE_OPENAI_KEY: the key of your Azure OpenAI resource -# - AZURE_OPENAI_DEPLOYMENT: the name of the ChatGPT deployment used by your Azure OpenAI resource -# - AZURE_OPENAI_MODEL: the name of the ChatGPT model used by your Azure OpenAI resource, for example gpt-35-turbo -# - TITLE: the title of the Streamlit app -# - TEMPERATURE: the temperature used by the OpenAI API to generate the response -# - SYSTEM: give the model instructions about how it should behave and any context it should reference when generating a response. -# Used to describe the assistant's personality. -# -# You can use two different authentication methods: -# -# - API key: set the AZURE_OPENAI_TYPE environment variable to azure and the AZURE_OPENAI_KEY environment variable to the key of -# your Azure OpenAI resource. You can use the regional endpoint, such as https://eastus.api.cognitive.microsoft.com/, passed in -# the AZURE_OPENAI_BASE environment variable, to connect to the Azure OpenAI resource. -# - Azure Active Directory: set the AZURE_OPENAI_TYPE environment variable to azure_ad and use a service principal or managed -# identity with the DefaultAzureCredential object to acquire a token. For more information on the DefaultAzureCredential in Python, -# see https://docs.microsoft.com/en-us/azure/developer/python/azure-sdk-authenticate?tabs=cmd -# Make sure to assign the "Cognitive Services User" role to the service principal or managed identity used to authenticate to -# Azure OpenAI. For more information, see https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/managed-identity. -# If you want to use Azure AD integrated security, you need to create a custom subdomain for your Azure OpenAI resource and use the -# specific endpoint containing the custom domain, such as https://bingo.openai.azure.com/ where bingo is the custom subdomain. -# If you specify the regional endpoint, you get a wonderful error: "Subdomain does not map to a resource.". -# Hence, make sure to pass the endpoint containing the custom domain in the AZURE_OPENAI_BASE environment variable. -# -# Use the following command to run the app: -# -# - streamlit run app.py - -# Import packages -import os -import sys -import time -import openai -import logging -import streamlit as st -from streamlit_chat import message -from azure.identity import DefaultAzureCredential -from dotenv import load_dotenv -from dotenv import dotenv_values - -# Load environment variables from .env file -if os.path.exists(".env"): - load_dotenv(override=True) - config = dotenv_values(".env") - -# Read environment variables -assistan_profile = """ -You are the infamous Magic 8 Ball. You need to randomly reply to any question with one of the following answers: - -- It is certain. -- It is decidedly so. -- Without a doubt. -- Yes definitely. -- You may rely on it. -- As I see it, yes. -- Most likely. -- Outlook good. -- Yes. -- Signs point to yes. -- Reply hazy, try again. -- Ask again later. -- Better not tell you now. -- Cannot predict now. -- Concentrate and ask again. -- Don't count on it. -- My reply is no. -- My sources say no. -- Outlook not so good. -- Very doubtful. - -Add a short comment in a pirate style at the end! Follow your heart and be creative! -For mor information, see https://en.wikipedia.org/wiki/Magic_8_Ball -""" -title = os.environ.get("TITLE", "Magic 8 Ball") -text_input_label = os.environ.get("TEXT_INPUT_LABEL", "Pose your question and cross your fingers!") -image_file_name = os.environ.get("IMAGE_FILE_NAME", "magic8ball.png") -image_width = int(os.environ.get("IMAGE_WIDTH", 80)) -temperature = float(os.environ.get("TEMPERATURE", 0.9)) -system = os.environ.get("SYSTEM", assistan_profile) -api_base = os.getenv("AZURE_OPENAI_BASE") -api_key = os.getenv("AZURE_OPENAI_KEY") -api_type = os.environ.get("AZURE_OPENAI_TYPE", "azure") -api_version = os.environ.get("AZURE_OPENAI_VERSION", "2023-05-15") -engine = os.getenv("AZURE_OPENAI_DEPLOYMENT") -model = os.getenv("AZURE_OPENAI_MODEL") - -# Configure OpenAI -openai.api_type = api_type -openai.api_version = api_version -openai.api_base = api_base - -# Set default Azure credential -default_credential = DefaultAzureCredential() if openai.api_type == "azure_ad" else None - -# Configure a logger -logging.basicConfig(stream = sys.stdout, - format = '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s', - level = logging.INFO) -logger = logging.getLogger(__name__) - -# Log variables -logger.info(f"title: {title}") -logger.info(f"text_input_label: {text_input_label}") -logger.info(f"image_file_name: {image_file_name}") -logger.info(f"image_width: {image_width}") -logger.info(f"temperature: {temperature}") -logger.info(f"system: {system}") -logger.info(f"api_base: {api_base}") -logger.info(f"api_key: {api_key}") -logger.info(f"api_type: {api_type}") -logger.info(f"api_version: {api_version}") -logger.info(f"engine: {engine}") -logger.info(f"model: {model}") - -# Authenticate to Azure OpenAI -if openai.api_type == "azure": - openai.api_key = api_key -elif openai.api_type == "azure_ad": - openai_token = default_credential.get_token("https://cognitiveservices.azure.com/.default") - openai.api_key = openai_token.token - if 'openai_token' not in st.session_state: - st.session_state['openai_token'] = openai_token -else: - logger.error("Invalid API type. Please set the AZURE_OPENAI_TYPE environment variable to azure or azure_ad.") - raise ValueError("Invalid API type. Please set the AZURE_OPENAI_TYPE environment variable to azure or azure_ad.") - -# Customize Streamlit UI using CSS -st.markdown(""" - -""", unsafe_allow_html=True) - -# Initialize Streamlit session state -if 'prompts' not in st.session_state: - st.session_state['prompts'] = [{"role": "system", "content": system}] - -if 'generated' not in st.session_state: - st.session_state['generated'] = [] - -if 'past' not in st.session_state: - st.session_state['past'] = [] - -# Refresh the OpenAI security token every 45 minutes -def refresh_openai_token(): - if st.session_state['openai_token'].expires_on < int(time.time()) - 45 * 60: - st.session_state['openai_token'] = default_credential.get_token("https://cognitiveservices.azure.com/.default") - openai.api_key = st.session_state['openai_token'].token - -# Send user prompt to Azure OpenAI -def generate_response(prompt): - try: - st.session_state['prompts'].append({"role": "user", "content": prompt}) - - if openai.api_type == "azure_ad": - refresh_openai_token() - - completion = openai.ChatCompletion.create( - engine = engine, - model = model, - messages = st.session_state['prompts'], - temperature = temperature, - ) - - message = completion.choices[0].message.content - return message - except Exception as e: - logging.exception(f"Exception in generate_response: {e}") - -# Reset Streamlit session state to start a new chat from scratch -def new_click(): - st.session_state['prompts'] = [{"role": "system", "content": system}] - st.session_state['past'] = [] - st.session_state['generated'] = [] - st.session_state['user'] = "" - -# Handle on_change event for user input -def user_change(): - # Avoid handling the event twice when clicking the Send button - chat_input = st.session_state['user'] - st.session_state['user'] = "" - if (chat_input == '' or - (len(st.session_state['past']) > 0 and chat_input == st.session_state['past'][-1])): - return - - # Generate response invoking Azure OpenAI LLM - if chat_input != '': - output = generate_response(chat_input) - - # store the output - st.session_state['past'].append(chat_input) - st.session_state['generated'].append(output) - st.session_state['prompts'].append({"role": "assistant", "content": output}) - -# Create a 2-column layout. Note: Streamlit columns do not properly render on mobile devices. -# For more information, see https://github.com/streamlit/streamlit/issues/5003 -col1, col2 = st.columns([1, 7]) - -# Display the robot image -with col1: - st.image(image = os.path.join("images", image_file_name), width = image_width) - -# Display the title -with col2: - st.title(title) - -# Create a 3-column layout. Note: Streamlit columns do not properly render on mobile devices. -# For more information, see https://github.com/streamlit/streamlit/issues/5003 -col3, col4, col5 = st.columns([7, 1, 1]) - -# Create text input in column 1 -with col3: - user_input = st.text_input(text_input_label, key = "user", on_change = user_change) - -# Create send button in column 2 -with col4: - st.button(label = "Send") - -# Create new button in column 3 -with col5: - st.button(label = "New", on_click = new_click) - -# Display the chat history in two separate tabs -# - normal: display the chat history as a list of messages using the streamlit_chat message() function -# - rich: display the chat history as a list of messages using the Streamlit markdown() function -if st.session_state['generated']: - tab1, tab2 = st.tabs(["normal", "rich"]) - with tab1: - for i in range(len(st.session_state['generated']) - 1, -1, -1): - message(st.session_state['past'][i], is_user = True, key = str(i) + '_user', avatar_style = "fun-emoji", seed = "Nala") - message(st.session_state['generated'][i], key = str(i), avatar_style = "bottts", seed = "Fluffy") - with tab2: - for i in range(len(st.session_state['generated']) - 1, -1, -1): - st.markdown(st.session_state['past'][i]) - st.markdown(st.session_state['generated'][i]) \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/wip/app/images/magic8ball.png b/scenarios/AksOpenAiTerraform/wip/app/images/magic8ball.png deleted file mode 100644 index cd53753774ed4e666c7093f6d58ca02a25be36a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 37452 zcmW(+1yEaUv&IP?+#$HTI|O$v?ozC1ahKrkTHIRPiWDzyrC5s=_YW@)z30D~$s}`T zk~x#?yZhM2X=x~8p_8G*!NFlEE6M4=!NIft?+HQxuG~?R(gH8+Uh;-svYvKUUXE_A z5Isj1J2(YTM+mPVMBU0C!pqIe2jLbG<>MFS=Yg|VxholcgoDGt``;6Oek0@sxQJvgqb>sn*PMd!VucI`msYMUC!^=LdKUcn zG3%HAzpmfP6w;t@`nG0JGYB4G`n_15u+NT9e&GQEGi-!PDoUaMy*K!6^#-|!cBFAl#g zs~yWDLx!VYXUHQY#NmJ_eHYN0BxA^V0Iw}hJ{wqC44>j`d?-vW2Qb~WRivrLO z$;rsb$jKk;APW+HI@$mG79HZjM2I~rNB{H21(mp8*q=XdcO9V*->Kg+iL7yug@<3^ z(%JC-=0Ww5QeTj`AG>}8pWhV!dm)~0;bjvxFDAm$UhiN3bi2Z;DQ$R@Zi_rD!N-T# z6MQ|vG_ZCN`Y#ZElY0KTx{loz`S&jTpYJv@lvh>#I+uj;as*Vn#ACx1kedXHmmR(z z;*o!rY7IK7kf{SACMsE?m~;u+nGwjfE`R_2{qXR>)mJ)& zoK8$k93MM|Vby#K{r5KJSbBuNfj2Tl5=97+fgiCUC?ZCSMr>4wucc*`dWXz(B@g{% zzehz|?8f*jH3hGbAhA}z9)aNW^fY{;s)`Y$yp21xv7e&sk1#YslrG79yak6#Vzvc#c(E}qLB!8{nYtm^ zcxh;0J&9gLh{wXh;)eWcF#BfpJvuoVyG%Rr2UBGWP!WVgx8yK1>=G9ZH8r(rK~w4nK zFp(8>Dw%`QkICGgCWu-8K_~ zexOksPIg3c)3E=zg5g)+1!xZ5?3Vb_`#d{%UK;Y1%OrB4^OaWAoXaAnq9t*l6l~;& zF}yT#g9bLIr!g-Xvk9w-q$5hCln_%P!w6xhxbSC=^s)X`Vy07^GF5&mZD!R>NA0&C zlVp?Be9^Tp9W%|H{3Z8g{zDy-a1a=#n^U4elP*i@Q4}+{N}vi;I#Wqd-ao|WSD3^H zKVc|;3<@gzT~JVfc;|58AxZg)QjVSKtfXpVLzgTQ%gey*lQT|Bc>&$FiT`dfJVxix z>^`NKI22bDlR9aV@3fcR_a)0v)p(NON$=RT@|ienn5!H4K8zF|R;-GV{Fwj_`2S44 zekeV89(M4W!UwNLk?kHGF;`HlNybXft+*{9=pd4YFW`qjxRj&}1NbxG#6@Y~F|`lt zy2P-QFtj%<`GZ-!z`S(o5VBcks}u4%zOpg&5awlOa(MYBrr8MIld#by!+z$b7%LvA z$e5e6GaJ+U?7%|mGN2^?8H^vet#^9d9L!8iC>sVt*m?FG9_P?xc2QM`XLHq@2~%n& zl=Kw?yGLLic4d&;zPX1*?XDGJ+FuDm1~n}p?E0HUzMoD5DSTxQ`-#le6pt7bP}cb< z#Lpm#uN@r=G;QlPZ(5m5kQWP(Oi*|_3My($baY9??2jJ{h=Vu7RKZsd$QIRsS`lB` zZ)DrFwB^*{v|j?0AEKxxQtZvH$0rIuDa0||Q57U&%lz8t`WO>7YynA<`O-!OFJE&$ zu$|o|&Ov!_uu-ns)%d|sBIEI{SbWZYcB&KlgkQl5vIkXP0i1lepmCA`l#q}1{ z3k1D^hnLqJ^|$r5rVsnA7|`fA)4m{51aLmnD`#d1|4E5^gj_&CfO~31ACYu&2W~O% z$5dNw7G4=mX=BfMc|AF1qa1dmlfb?7sD{{*)}}8%0zI5JJ{11;Frz;C55}yg+b|ko z37b?=e(=&C*b=V%xv_yI9rC`KUoruLBmn1#h=>>*9Q^(J_Y#~*AqdzDkrm!;^Q#N_ z8Icy+JN9(q6vx9Y3C!noG|{{#d?8B^%-nD^s^b2O0j0o*Qmgd&NYGx?IV)EA@<-6O zpUu2y%rE=XlHCZD$bv#VJn;Nwnk1CTwJ+es=exGB{q3CXz$tY)RbxpLh^^f3$D2c- zP6qQh^EqMYHSnhXK*h{u%FQ{hj0g+-9JcK(hUt^H=MM_2l0ro*u4I@iGF9q(xP7RF z;6G}I{KZg*yHN<&DoW4DhBiE_E7lZbzQ#=?@!ggncr*UJ8qh*|K0P~w)j#CvXi2d_ zY`5NphDvP-2?%)KpKtuBMz#&Fg+$FbmTr$hNN~dfoYrhL^bm9w3bl%|Ll`@X7asPq zU|)~F8X9DTt&$GCqln=|s7FoGgkAs6*W)+({RO*Zgo{8Aq|hXViT=i0LByN!m6qLo zTy_hL&Pr*S672sXqvY@X->QJdD|`rEOX63=kgYxvw?qjg_!q;SDcJDH$Pkp2WPU#p zu_zVh;XxAaP=?fk#@xnO7o?!4dpam=+N;@Yh)-f}tD~si0($3Aof=O<*}!@*i8pIs zXVD)rmQ2~(+l%>R;!69)#eZ4`RDo5E3<8Rk@udU<-hjkWbP|3jIvRy%o1JZzS$>gi z!PGYvC6O(8PeWJ5`&QEKxMO*(hl>Y2X45zNe|PKe%OjFv40 zMaj6po2e}(bAw3-_A-3p$3)y&sZp(s8Bc`vT5=D2V6N(csn!$Ut(9wr5r}+9AdUic^k#s@xQ2h6# zoco=*IkbYM1z#dRp}m3LA!}uF>!Bd-V?`lQ?zg`hgpmvkR`29u1*a% z_DG;a!zfL_#j~t;vT3{&xhPk3v4=Esb33D1qmox#mP3mX7A7-6-sjIOi8=JUrFuw= zR&(SI%T4_Auk&! zI+35@?b$~iQ#C_^aphetrwN^gt?T9CCwE$gflr>CdSHooPYR_7F;5Wl1H2U~MP;80Oeju0x#k!4lk zK@TqLow&QhhNP0#QIrLXWc;Kq{w_mcgi6@aQf9i;lLT-ojy5HZCHMO(*pU=Y+}cIo z>MMj_)TxzT5dV1+;X3RkTI=fKYmLq65{!oKq8O7tL`|ZLg#vxdRQE16Tag4tf(W@h zTWOVs+ywDXBOeaR!S~mjbO;Ix>L6#Gb<>`<`RrkFX5d8a6?sQ`90^p2<_B4i!Ahrc z{N&s2joiw2mbfLiqTaIWqC5V0DRjxjDHb8|cHKqv;6W-PlMYF_WXch>V5LeItL27N z@4#jEjb`6aNR^10bveL}00sz0UtL;i2M%V#o5rFsDyg5_d3G%SB~=0Qd?F6o=8uu$ zc)bY>zWbHAmN@Y+mJY|Xdy^4ldPI*T-e2%iAX#qR7tc~sVJ_K$yuD<|@I^QxD)-Km zdTv&xA|A{#{_saJD#&f4J1T7;OlA!UWVH$(qL@PE;_5mIAr&~awBv~nn^Zl2a?;@C zzL^u9Zw7fPUqC zw(>`xsz^ANf6=)$X&84G_YM9>>W05JP>W)6fwLsl=J2~7uU%J}00CPy4W#Dz%Ea-H z-eekWALR7&?e#ej7`BI=3y!4w+6Xkn&cz}hQ4}D=zf|7Skc0!hHU@_p1tAYXPcU%K z=oQZbE(ng4^54PgnG1x;ns22a=@vRxbUgs8F~@F=Vx0urRX)&i>m(t zYvE+3ljRXOYYgBWh{};OU)kRB2$ZZpoYJw%u+3~Y^Y$j6t)iyoT zyeZX18N=;tEc#*aS& zNe1V3!ejlI#vZ#95fxkdgX9b24jmknWXJwT5hVqM?VU49d1)Bkxh4hyj*3(w)-UCw zAp9Yf40j%F@7cYkNi_mKMpTZ2Al6y-L>$Ms-B}MYks%q{zxU?x_qH{#d^+<5`kC4k zi=jl3PVU=bnpx?m?Hp_okn(!;4Y!V{m>Bi-N4=Uq=1>ii^S#_8PI^}M z=p|yu7QZpkk~c#7u`DOCZnyAv!ur;$Y0<9(qoPxnepV2o-`<~zx)k)Zz$ z>iRoW%s*t-RdnQSqJQJ+pt?0j56-^jt9FpYy`p4DH)>QD7@+gZT>lR+2 z2G7a7=|m2DbmvSNE{UUYMJSZVql&;a2`A8XW<#~K;iHLF{Ee$NRf9!86)WB}QCKP~ z&aRuK1hmo`Bz#7kI;Vr{xp_So4wmqf;3zLI_k!WUj^?VWs_ct{?*_sVk38;JsFe$? zKM85m9Q3YKiU={}p)F(w3LsRsq!N7&eXE*~>H1k+>ej{N?e@)rOGG*?+@`q&=cB?2 zAVSc}(fT4Y5umA#x|w!+#c?jgxSZ^4BW%Au|F6p~|N71WpZQ+0d$-=m-9 zbI@Y{1JZTI6!*5Cg24GXW~FpaDgA^7PTh0BhMrebv@>|mJ=jgq_b-Oe8z9PBgR{bR$y1^x7xgOcLL z+XPr&ChTTsXUoQ5d;9u+)J`w5_$qjmu*(e`60O!Mb+rCG0K zjw-e4vJ3Y9h58#x$G%0Nqmzyw`zg;Lwc~dW4@RQ;UTk6^@tTs0qLeF`c?!WZeA(kZ z>>?<$#Fw^I2Iq56C%QS-^{^f?=$@HkO~~R$sZ(gX{^&%+?UwSIs5d`~!n2s9jItY) z+@tA5F|}!ST4F7-!ep&(?7OYxlJPZj9s2j@S}P2Eb~2ho4m3qdDysL)n9@o-v3-$5 z=&$gv<96XkG7}x|;6du2t$eurJ9t`51zdv-p+8N|4^`%SuS$Qpw4=eX&)-dgNj}?F z@X5NxN~PEA(Q&?xPd8DqmnDBz_s>c!;Cr*Y>V)rJipC<(&bs^8Ax;3Hprnk#q#z?J zV~UPv9)>)fj;GNzRSY&Gg)?`m%;r$Tp1z=^>b>abh@%lbWxuO@5Tko4y&9Z;HoH$> z(s5gx*%>&luf9)ME>I_MEPY}cdEeh-vCgNKG$>AdW;pr%>FJ4nN!5c&59!3(5=$fG zVLK)@=7;zV=DCr^Nhpal2&qWe!#X6S=<(+qR~Yp>kVEO#JGqjx?}fJTE2C+!IL%{T zY-GO#fvK~ZzXU5krlYKn(nH=clk0@|?e!Bt{gk;$sJZbhma~$ zs%@>c^TBe;y(sTz@}kn`zMbib3?`4Y?e8%;23%(E!J9q)H@C+Ne0+SZt*z_Dm}aBx zxMpT%O+|oOL4)IHSP=FZzq)DMA_GyIQ91(GnCm_r&Cj8S(5E2pMyKL!b9JvDQ|i7+q&$W0e?V8$4NoVGY<9X zY8+jbIQ#rr)%<$-kDPx}VrIu%>DsNuy$)zJpBuWn-)qYeycjjn^Q+ECmbL7A3O4c! z@&YP^iIFk44JG==d}^;rHWqAmcbAyU9L6~rg$qY@vsCW;)$I>9R%vT%#zFCiP&P=L z(j-iAiY(BVuy~P)NhVOLo-1necv+YD^lDQ2NPjdFaVY%{>&mt{cjgHDWbovg&!vI*9ZXK@Dm4w?&(ylGQvIk*(rsceF49AilRrqRsK}5@{YIz8(*T%j> z5_0|`6Fg`iVd+YI+DHKwgfb1*5iS+TOCEj*^tEBIZHIMpRr#P^X1$B885 ze zGpa=_KOFA_iz8Hn-qXqzwq7#|&F=stq$y5Mcs{Q*WAhyA9lIp+X7iLT+f}}&gwTJo zB;h5r?m^5Jn9m0hmhPd2UH&(W_u)=1E@mon>;fPfxJJ`11ZP8^+U!Dt|Ni^$`Tj!D z7cObugV6-S6<2VJSxWU4)P4-VFm2fKsI70Wc48FRoDHslcDikw=0Bq3(O=s;y1=TXG%RJ=8&rd z3tJsQcb_n<;n78_P9!H-K8kXzVm7X+4J+-#5e0&4F#meh{8FrmbZdJxyv^K+;K{&=2g5!z8^&vt;H5R6`?vI*x;Hd$ zn;MQ%l!I*<{qHx67J)`Re}$ZQC%T8ZwMBl|fV%11JvytS)cAIC*Z+OTks5YMp#jkx zx&7&zDXd>tN_A}g`C#_V=f9QKnB#dhvLlEj30IPnebrUztVDKTT^sS!?6}6_d5mYo z0upF3xIf3)uYl-FXiqw!h3B`$UfXwddy$)rF-0nc%!TP$9{(6gZT4&Bs1RaGglfu5 zgg_{TsB-r2-*Z~@VU7@f$@?5^L<=_W@oVny|L1BfY$mGJoHk7q zai69&j)WRHFx=`Diy?dY0`uxD{cI%@!QNEP??xKC> zwxYvgkGK0Dw+-beYJP>?)LDNWGMjW35Ur0v(4c{LtyI#>hrMo9F(A zy1SCsP}bKMaKt=qp}`tq)@BpZuW#6(HV!OM%_7LCT}H0wbCbsM0U zz;MaEjb~+!BKn`w61uBwL90jp&*VCNB8DG7T6gc2QSIYZ z)|fwY`}&jWL|motG;Pi`gnPa>?dcnm4R2IgLqjqTKX1;szq}U9rHGgq#$T*^@oNY$ zD;R~*4bY(LOvl(MNPF2FCO#LN8>_4FH;d3Nx7wrg&5*WaS*3(6t=YiVMMDdID@#R` zwMteu26f2~nSb>vdLK`zY0uMN@k3b`$r-q^>M8`1&vl=oyk>WaR~fxDP*iz8v$!y% z(h}cF2S-wMxNdaM&d#n{ReXiOFew1-si44SP4ex*hAgnKY+JG@ac$Hhk3Tf_TP7ZY}89(57&rW^Rh!8s{>30vi zLk!v2*l?h#DAOiAV;Bw>UZk}D@5??VN9`Eh)}l~u5W(*9aPQv!@TRJHn;#b`wNW6= z$*bv0I!lxZK5{h1uQ@jj>3|qs^M7-?jOfOkI9tZ(UmRP~&}wii9=h7#lI9Nb7j#hm z=IA=0Rpy_9bE*CP?QL5}$Ia}I(6<{*^9&O0j!-imVDM3d`8$qK z3f|9b@}0veu4iqkRuKqqxJ7|xRTCK1st7H3vE#2T(7i+KF6v6s^>J=6nXxV-#luKm zpvJ;3qm>AZGs!Yna85|rfM~K@XN1^n<)r0=Jk)Cc!~C01QftKRn7b>%D2?BAsceie z;&-s53lbx1c>Y?Kt6TTYh3jO%Zux5L2M_A2hVbhM`oqff_0?`Zxw^D^slQmIPcVAr z3YyJi?--U4mU*YsqB-tkyot)K6UhKIH2GLD4THIlGMX%w%4})Mt~44D5h@07GN8p1 zf6B|VvQ31=dHx$%Ue-5^oP)2Mjw+>zJCk(uc8*o-#=lJ)%vhscbu+!ohm$c8@SsM! z3hBr@|1;cy*3co;(@PLjNNlE0;9Bu^{s^v6Cn$6si|*S;?pb4MLFMj%Kd=_fq)#%@ z&9NGcUh@ zpjs__g(zY?*Wh$%LB$R>mF2-Tj;k*L7W?%TYpVLMBFmc9$U~`)`srS2a5m0I&13IO z^K)}aC@6w{%P7ech)2r&UYCUy_B*Trz8oM2uu~3J1;~&n8po=iR7!91U~4PD-Ua@; z9*g;-%9EywyJcj7DdIsJ8DarROuT=9H6?OR<%B-|AaoN*gE6unZo{Q7TVK z<&BhtHtcSjJ9i>oO9tMZ0Xowc#N0MLYM7Rs6>6CT&iv5m++&y3ca&I}wuSU?^LYJr%P;)2ETW~x@?NTH^S9JTtsB$&!2vu|5jNw z{utoH0G|XfNdQWT?2BW*%iTU!&+P4UV9FetafMwA81HFfaXEfj$5iN_{s zImPDoghmn6k$b7I1?zkr&*WQe{*bA3+whv!pUs{$Ow%a~cG7SV?s@$%1Xl~EjcX1j zWfU}Bj#X`tYiQUi{3@LpP0Q&)0JOYzkKNH2-2b6eZsc%j^j-kG?1pLW1z~KBNn`=- z$&Qix2hp-QQgjuTI)SGLG!DWoIe*S@k*Ww%`nO~tUvedVSC=HTz+9lXt6Ploc-~7% ze#qz{H*zr{g5=7zU#D5atH2?Ux%RvFM@5t%LD-n-6I?}I>$kvr&P*MU>~H<7p~@OP zeAv1=S{BOMRyxCZ*7q9z+z2g2mVJ2D(JvVNarfUmlx9u~-Ni)Tp!c>D+ZKBay6ApY zxk3}>HTtf{V1!`AD_LJik|?L@@nk1A@h&)9qE0?<^o&L?F5+XIS=bZlCaaF0qFyBzEtqjEkKZ|iNH`4U|xBu zq-L=0m=3`kDygbsxQ~xI=uDxB+&7Bp;a2ST*iw*~AYM=^btR=PAjGUzxwn1}hFm(v^ zc7$j-qorT;EGDKy`aaigAXFQDG))6{+OWev>Py_UT+V;d*8vmZ&QcjRWgRQENHpz& zFZ;|T3(}W>e1d}kqCqT8uukN8=~u-dR&bvp*OE3dS*ubn7%)qpM$jmv=XAZ#W1haQ zz~uNJOz_0+*Fa<%QgHR@l+8{Xrjg3tVliR z04XIRpShpsiU6X?<-qE8}N$REtJ7UJv6z`3;0%N+5M6 z9SwbbMW=S1x)I_9HAXh!**Vea*2R;$My20UoIsLTl!OPOoe#sq!+U%5aLR-V>$k?l zcHfR71@h(~X8F^a1rI*hUiUGV$!5&7zj_A=F4k~kG7Rtn^1Nd!x>h7|id17+ZtC4f zBN9ReeoDXKAcML(8rTbo6SB#09h?FB2SLb>P|n`_?y2Vg?K6T0aSqG_9MDYJ7&qaN z>BXWz9A~J5G{?(cA@)T`P!Pz15Cum>cDK2UA!SS`rxm7Ojrt! zH>0Zr20q{Yw{|Q=u8hHrmQi%0P}g-9z$S7LdqV$CuAcNK=(p(svu4zJqV%Hh*(jw2 z@igPWm@Hv=I_>kNhaJ8KFEMF7;z{sPhs{`W@Ug0eF=hl226WLfARvD|&772?yo`b) zthJ+qxSRPAotTYb76oUMs*a>Mt!ZLzz(61Wk->Nz(!Q3xsKyLEL^_dvrKF^snVBI` zif8$caPD&w8Y{uZ+75QJMzjLc#A3*Mm!-qcr%?FOBe=kUZ6N*w_`~p5%xvN#?qCrX z;|_sm>O-%m-@Nlscl0i&jkjMHw+;Ph5{B+~+L>!!4Zhkc6DhfJxe{TB?#)v^++`^^cOLmAXkHvPs5!t!Y977R^-taoJeG= z$p=zGTH`Y3Y68yQbTLMz$va*isylmJL?cVjvF0=!0t8RHBp%&0)z#JCzLBr~8W@na zu=oSW8CKTZ!x157Du@%~>^+u)9Y&wxXj%Ccp5Dekoxhe=*Nt(cV~h+>&FrPO$n(Qb zT;k-B4HErnKc`Nc}pd{wbAg^=Ys{^Q7$7lkzWs?%2j*v8j;(nze#3OO&UYe&Q+<#eCvQya9 z(cKqBdz-_@+`Fx@O-Hfc#;ZYhfYog_+l*~(LEzAsdGEk8GOr`S`4wLqq*pNr2ct1m z@!wZ%lHbb6tw(KsK)}GLr=`?LwwZqy)%Hd0WdGo{S4sXLVqgYPP2~FJ_0s)5Q@|Vu zGah+?D1cH@6uHtRW~|%m)8F(3^MiaS5;1(WZ%t47mpz7EOO`{1-N;(!p>T7nzcNoi zZ!3KS5>!KmWcw=K?USh#3AOpzm;`l`yZWz>W+BFM%rTMMX&*37Ftg#iuvYv4KfTF6 z_qA{t_Yi&L**q$Mx5C1YMMlcHa7F2{;i~!l{T1l?FWtIFEIN~Y(Rh#WV8YEq zrRUgMgK3xN_1;9eVoH9zN)$vcNwZKG{u%ziL-U(9lw$&h&)a^0*8x$y>WV9%X(153 zK1FX3|Do`vP_z^(@06$Jz4zO_k$Ht95~I1)9s00vi72qJ_Swp^OyBe^{qWuLJL07w zZ2qugX=o{JuiaF(Y7VE2V(-U8?B~IBd6LnbnXh#; z%lE3{t<$p$8NH8Rp(lB*udfFq z5P_03Des-4)8K2D88&Lb<3_f8f_YO!|BwXSl~P7W6+qwsNghki5rgG50S?@5c$ZQd zOEzw{)wQ+33q?gmfQG7u54gN0=4WS0UMa|Qnf;dqO;bIGbcf373cFtM`rb3NVRm8D zOhC&)(1zLpcqK~X?XHi9MZtF~=+Pxmu!g4Q4`8~{Mrn(ia?{}B)G%Z=cXx|A2aG+` z6py?sR(s@55Z^cFl{7y!Pge$!ozhE^g%plf!maoCf3&x^*VSDB1VJ)5oqutTniT>TpjKgV zvArBsUSqq$GNqhE3*$gqJ?gHS$UlbNa`+6Kj2#iWclb&C3N^S|_;Lo)60q0xwKWr0 zDFY}E_*)YS%aa3q%l@J~gEXmi_-PvP(8OB3Yy4@sy8b8ZAh2VkYf6W z6HHX#I6(OZUBjej|3l}^egRdy=Z?lJP3*22>voHwGIlWNKXlusZ~G-)?% z7YQyd%+HUo8h}&Mi&`sCwv>FNwM7_>)8;)Fb=}Y{9iT-Uy7hV0nH@jn##QKWzaXoz z2TsRYw>O)TF0HutaGw9L2lMB-uE|;3(*Wih8p0H6sgS=X@6golp=|wi2_H|uKawnX z;eXnfB@P6jZIlK?Z9~5Xwh=%xd2oPx=cp~7LfHn|CaK=w5&X51hfjvAr8u_+II<1j zRDD29>3m|&zeBUPhwt{wT%{KBZPB0}bdEn)8DQHFQQYjd6J1F0j3<|VdhzbT8MEnV zp@SA-VA?dv0wCECMo)4gf7plSF3mPy(xMuGROu&DN8~UXZWP;no6bq4MMr37LYJyS zjdLj`A)#(%m~Ce$2m(|pESoTe5+#wmvMsJo#t-xw&c>UpjnimWfn}Mo}phb>d}8GA9_?kn7w;*WY<~=yY&4Cvpt( zyF@R0sC=Ni`r%*fSzGbzUCRWX)?kR`E#}(PZ`W>nAZ-f=-t6>1+)L}vm_nqHM5*J0 zA@t*g$n@&%&=2B@iHf4n?%adwW8_RL?00k4?+?55v`pwG8MGCD?ryzqaUJadhko|A z4)o*--#(hK`7xG=MzXgp!hXPB$YmAw9wC(*S0<M z6YCIOn|eO;&rKZE z$fYW;W}+63&}sM;A{`h&&-{6ZbLjj(ga4BJhB08Y^HT5&2~kS~5CpJvp94*3j;s@- zXJFL!qqDJdQ;2%5Ae2@Qd)-?E_hAG_En2hQ_*BcUK4q z?tk(p5Q|Y)jpu98QyEC{*)~*N$9#jn`robC_usWQ{=K~&k9YCp;ViLQj}tRD=kGS? zbzka`=QMHyQI0pucjxPxg#48H3>;onEDAtOp?02&G84%mKnIas%XT}hW*~VMO zfXNDH2+)oZ=GSn|r}uhnJ+}mT{nl9{H8^Nf`?T7UKR=87muEX#^gg3_hSqFsiAET} z?=t{(`Kfx;0SG;wx_(sy!xCjYMsDtDmS*WoH0R?Ah1fL9AKsb28S~U|PNa!ta1*uI z4hDQl<>7Z=WxI5u!JMB7N92zq!+! zP#r~W1s8vm;`lz0z0=XqfSvTIxY*6bWure-ia=XKp+N7w3dN<3&2yH?@N@=tS{vmY zac8&svN`u?dTU*r&98l{Mn)CPwCco238CclkpdN2@lD2$1kO`a`3?amYU=02IsqN66{!5ybhA&I`pxds2Kt)x-k6}CNGR>0gqWMMNS7Ec_tdNaHjdgnB zzuYZv08-gH2DC6wqzk_*=%KEp^$6M-uGVz^(-$o4x0@scc+#%C&RW%!<0%8fMz^Vl z=~rk%*;>n(B6p%Oevt)q2D_=+8>Tx*Ji-D3S*_$Z5vd7c|MmztuH50Sk+(fN(2`TL zKDE%Zs!4a|!OgSw?G@#mMcTf17ze~9z<-)O_S3}-p`_K-g@y_z49}t4;GuI?_={PQ zMmR?j{SO$6Zqq_$tL1CNCC{r!_^KAM^=Z;bB8olS9gwAtS^S7$#%xH~UE1OV-_t+k zVZhcuXU5mqwt$UtY@3l29j(Ewux^D^qLciVEn9@xYe3X)g{0;MseMU{NP!ChDoO66 z-1@ON+dJ9=#m=VFwp#9q^p%qG5-_ExSFH0lD@Ys~U6H+iTuA9(&vGrRjRWiHhoFk{ zDOK7zf<}X7t{i!j091_sAX)Dto~LJL&3gj`G_B`NhxvJVc|}B6+=)XIZFSNzu}aJg zb?a*PwlQhAvH31izvYIJ0IdX=!z3L6yHpeCR%N6DQW#NYgI7zA zX%&qF$wD7~yWbxDBsTXeZEn8WO_5@B=6mGN^!sD{qdRotxgPO8{jVoR>XYG=v`d;J zIO6X*P+Zk1{5kY6^TE2Uo%pQAc)onO3GuJCeqPl6X~Um#RFDk$G}eq3>O_Iu9fybDd2aV#oMtoK9&Q0oFbkf6CC5~1s)WbVco{NY zPhE}QL7u()fUS|%$Zqiz0~$auyd7$4N|XgH99v@XUGpHWv=Xdo)_=8YK#RQ>?2ny) z@je&CdY!BR)f!3i!O@YX`j>o$B|XzBZG!xa=e$$7xgFFgmEFz1&$_FTjZ?I9g&A$4 zKe0NTTiksM?tHJIW3f8Xn$5Z^*Lv`XBY?TSv;+H=u2i*jHcbEx&{z#sZPlt`PDNxj z2-6!Ny_REH=ik$PrPJV;G6Yogf7;H_yQe3Nm?=9@@Q~!Brm4DuMj})o{}D#AtHr)= zS>f=9)$(0UHIZbr5Pm{+nTTim#~`fHinyV$O_y)Yui_o{PRFLdp-9ul1<>}NuBW(w zpEx|bqBieK%%XM6)H`EF%vSLx=iq{CS)Y@&;9uTSw$BVEc-V(GfV zOvi_n7MHcJ0?y0s4YWqshlkZG4QknU&hd02?bWk*ir1NhbXLd_%bch&Ri2AjuU&!n zu1A%+y1Gr7C#R>%@@4Adn4KY!_mH8<-Z2p_BkF5OhNeY~Pi-RQ7>Fx&KEZ%VlLmyN zp{`VABMni~n+hfrC(KN?Q9P%V1U(yvREO}TYmpclC_?uicf^%iDlkQ-t^utDz-J_6 ztRjLl>54W}`7EEi4*G6;eu~vAhKC<8Qy`;cPKfnBwPv#}xj<>ub+T=B$C!_nxI!PN z0ELn|=n@{kQ0L?Bfg*1L3E4i#;P2*fuWgVme)nE)ehp?$++q z`IVm=G&GE)!Z7uoapWLxF`d-S?zp+9f{5>WbkIrZ{8$yk$woqXctAM^F!3GK0rS+0 zklm(=(dDUiwT24OcMyCyA0ZQ-1Xph1YIE3V-b;pM^;Q1-9EpWqQke}r%zSSC?Q3o# za=DvDz%hf?-uO63Hy~b{RW;lj4ULnlSO8uQursCH|2cstpV*T?)^c~3ojKWlTv>25 zh<3Kk3UVYmjk~T@%f_(uo!OJdmmJ!ZqaPDXnyqJ+cy98@qGVPm^XlA~*sONH;#dJl+5)mtOBvqH(>lO!f#%+-eL`ED~!Q z8ykSG`TE}fFLQIg9``=tHcEB50d44_F6u6IJ_eie;d$)q%j-0{^jzf z=*sEus)6PoxAavp(<31X39Mx_RpG@^JjkAYI^$ZB5v(*ynrOR@ccGmGsr!1NJM)3a zHQL;P_r2N{Pg5?M@IV`cN4n07<+#(<=~1;un##KqhA;N&J7;m-t-5A zxCCOZzUxW0#=a*fd;4l_z}QuX8Mc2hB;>Nqb5zUzlBgot_dm^-xe~AiwCX=+dL{|$ zIZSMvQ`&KD6gbQtdE%&%{KjxX#YQXURbg4kI8@HYjLLYsU%)$VAOUru2_mzhPD3MZ zh1rJdUWXF9gk4KCT*SgJGGx>l-QLkko&NE5ou5V z{CoWq`g&^dJ=-vcC}89$vJ{g*E$bz0OCp*A~Vzl4~)J zCw(faikYfhUGA8i=jHFT4;ju_=hBJ!;y5zkovL7%L(IL5<&hipB-Bu2D3t2-d9}3K zcwy(9ycvmMV$nt)M`a9%7WLovUIYW-CK9P!&4>MbuQb*O{GYcKs}*R8LKWy#LBPB_ zx-%SK7F}Fg+IG=Gpn1=e`R8%g7wCa-8tS$b76-bYwQIXZt$|&KCT**uvY|GF&evRl zh)o$x8rCv~yr7*~0=gL_+OA$%7RX|)%Rbi!=usAl?!_*ThLn`l4~cK>QUe0W8|ZmW z6bp8x!i`2$q(_pj-mrV4Qtkxra2>$NT3=kulw#5(Oqfmt0%Z|RaG{rx)c=e8kta$w z9|o2D4p*z9PiCOR)*jFHTacOQ$1s{V=ZxM$U?!_CwC;9RJn1CRM*t}egI=I zwXnd+$aTxMy%5!F=~ma^mB$Xo4)z0f3uR?x6Gi9HdIVgCac&FDpq_3%69QhJ_lWm< z?tC4*jvmEpMAq9!Od=q*Fx-t&WuoXR8-$)AEmZu@RE)o?P)o&ca#6M4>Qp!JtsT$b z(I|?_)1Z)(lhf7WI~l-+e@cUQOx$|C52i_%RT4!N8YXBFGmOa?)lbpcJqFSB#LCWr z{zk%On6;i2cF|nHaBdq95f=ltN2l<`LBgireCjJS?A;cbJN0&Ya(l6}X{Hyqx2>ui zGtNE`&A@uf&5hBuBRn7mcs2fDhoK=l?a#nAA?yUCs!Dk#C!mj zZ@bM+FPO1oy|UxlZ-RPFNNR9?z#0wWRwNE16B8+`B`=~To4 zE*>}HX0ywh;CM3-?*RA#4-)1sGX5#R9)ZxdA7MvwMZYl@GQ_~naCcx}08m%EySwi$ z{%|#j>jbI)ifu}&`3@|DT9HE8d~|679IGnSK01y*Q|Y2RrrBH&$SZ{?rM*t|FrYx4 zgbN*;7JrI+PK-#$8u*t2NeuvI3B7&$J>ZV{ri*sU z2WeOR6kFZccm<*@)Z%_%R3LhSN~yYqffAc{2j^Imf6@2M=|y^;sq`(u0Y)&QJTBXV z)ek7R!>?id2H`sJ4J=aWJZ0Qx$H(w=a7>;HC3{ltKMasVu2eUx97!GWVo5%&tQaNc zKU;!fww2bBj;+IN`|GSukN60sYrdQa8~vO1vY)K6#Fv1zjxccl1TRAz4ZXWN&}Ve3 zHA*}366MaM^Fp>hpj)D0oKU}QQ=5l8?gLA+18y=LDm(CPDw7kGVm?48=)GHY4&pc@ z@h7jO-R-%WZEkLUdkYyLgk&ly`JOi$Gfeb+3*wVRw*sYwyuJ7vZIDy{5vMaJ`m_f# zdePabgS7VKnyw@1_K1Tcf>jD75A$s&&7YwaRC0BD`U@dn=`rv>n(UvD<32hfz+o#J z-_ui)KoG{Ib7j5g_0@C|*uw_b*Vh66fk$weoti`1NBG7EK=p46Qm+fe{~n7~KYU^_ zpQHb}oG;+)&Lclg;ced{CK~+s zLjsuJA|Cx1`+kaiKV(X?mGh)#d`58lZDd?p(`xJr<5sIN>BeI%F2r6Tdy{CQD{AkX zjNoIs2d}O0v$8kcj4aJBUBpSWx%V0YSKE?&2)=?{WHC{N%cMN*<||o0U{I&ZCpv+V z|1Ny^&LP9-^gOT1)%&oMAXnnCj6Z212)>N2M;;!M>-`loz0 zTE#90GmoWi0uhs#ppw}-#o?&p38sZFQ3I(>0?Cczw3e!vI*G5G9Gf_gE;Q~A(Zwg2 zpazzj6lH|~Q6B9Dqi!tRPlNLc(BS_h&zF)m+<3@pOH0GWy7hdgCs|)arN<>wII711 zl%0|ppZbBT=Mtf9F1=AbGvfn5svCQ*Re%N9yyjD@gS>CP7ZJEyzJLI!b|tD@WgVtJ zwC`iAgucx%qMDKP-PSB(+JAz5mW&{J|Fy8F5=_DSFu@T5_RKdtII8dL1jpNoYP^{$;LYiy)k(K+OCDzb% zxd(}Vfh&j#$pf)hE!zfn5=umoU~{Boi39*c{Ix-Xj1Nmw5h~+hT-L&K0}&Z=K$2qU z6}X>J<rSdl41Zr5zHcWt&WSes}$8s?QAHqFb7Nzg6C&|u%mdkEV-u71 z<64U$w`~nQSz3b~KF=lA{L{gpN+xBb5;4_0q&=M--OS0}%zwBE)h3^~#)uH)*^ugb zSnOW}X*$b4l-RV?F^wvocbl+?aI{Z;DPF%AT63yj!@$_4p!XLhB}dQ&`+$z-Rl%zI zAd(e-1=P>%wqoW@ID@=E*oEZDl($DCAd(Bo4iUDFKGdz4uQv9nVl+eYv)Ni@ukAWX z!c4*cadZxTb-!&K&*sV2$u?H4PFS{=v1PYq8_UKzxus>>T<#Z_wT$I@zI*vmw)y8gA^UsC__dK+eBis$+fN0Ts6{ zP3~s4Q+^KvYBxWNUGb?cy4oF6p7+#cHnvCGnw_LFbAjni$cVqKW`>1o8Yc4e(?hik0EtIyZW7KlCa!v zP3mpsv5X*i6t2C;3TT?p%R7cDlYKrM7vmD|@r+l72{I->K3ibin}v`H?jS-; zl5rIMOXUdm1T--iIU_Eb1U}Ps?)-Q&I7|}Mybg*QjC5eV4%S~8X@oR+RoRpDlo2IA zWXf0Gl&i5Ff+G``t|(*kCop9tMns(bWHm&04C%7<4<1r)#>xQip6mh9j+Lj!dyO@p z&~4P`jlQ$M z0FQW3i5h9Wj-Y*~LsnMcJ6n+a zEjKT(>t8bX(pZic9&D?uQ=qfWZux|}nPGGOFGsjPB3Fl?2GZ+i$NOi&ehJLuE}odm zBGiIO3kEY^|4`pZ(j2SB*GepfWToFdLoQbocdQ8=tEAoivK+Av7RTTrxxdE8wb*Lu z1TV2=zkVw?W~wqv&*#^an%D?TXrw%%5pYfuA)rT%A&(E;EcT zF3cWxP|O7I#^Rz+W}vvTmj>cYdbB83n=;T;J~sxBjeM*gD7c`eT$z|9r}48=eC9ET&)ps+PBfuyiq>R*rZmVk_9zaY$N|Om z%!^1){&Lp`)U;CkVIZ?t(%eeIHfg#SZVN;NBJ_L%1{EkKN7<%S$>FvGy3O_&psoNC zYEy0g)jRo)>O80EG*QY9k3VUn?%>m-EmGeO1cgmNyQp-4Pw#SO{IHmN)uJfKLibu7TMDGN!QAk(|B@YQ=bvP3Qc_pgz(~`5eX15QNF9$jo6Lx}l z3TzFMaQZQDc$Pw#b)?ht?hnI*IDpv(QxfbamsNh@E*1Q=b zyZBj&Skc%(nH+KKDJ9J9uI5fmUJI?41?!Jptsy=4CPwwWjgY+yXN@Tf&WI3rR8O29lrWK?hw`|*oAPtr=@_G zrMuq&RDR=CHDzDH`ox%*sIu_3d6}6HgPi`C0N#b%qBpEtexFvLTq7|3z2^nY&Hsl? zVOIoBe5_73B&fW)ZaN_<0jRbTd8EKTogvv=9g1yuv+p(bSbZ7wq1mzIKZc~P+u&(l z85+E^D@zm~9eRXNPk8;7zg2&t8PtW(z63oD%8iE^5k#l+z=bBEX(&;%T2`(U?Qt5Q z6zTwpw}6Me^dsrSBV@5x)>f6ry zR|{)v6vo!@IA$|++Yk>X#9!i1C%WB_+J^<+Z{=1qGWyg=W}I9j-x{sjen?lT@vASV zMtH`?qepP)H@AY1|EB5+jLA1x+uK)mb{d)722O)aE*Fpcj0>G)lkkruJ#Y`3xNcq{ z)H8K4pT$0{a3C(2TN6bunM6yuMm3uS2SPq<{V5|by#REC*!Xy<1B90ub@{ohAIz1K z&x5(Ziy^_4Q6(qv9?l&w=MXBAqFKSK7Z(@Bis|`{{ujN-Epe@%fg}r6>K>^CgX#06 zlfN4>3+4Fzh->@fGSOE;jQPj;xo^}?Vpg=18$gW-tev{E=GKr zm*;=qtOk=Y2My9bpsww<+dj^voyOT(n##2I1m4))8-)BsiirN{3aV7c2ZAxwGXng zT?vGl1D{2bg2$3Mg%z!&gq|H&n@&0BAjNVc<~D@4;o4y1*af z#@}=Xrr2ppnl;l`$x2CnD>#!!drNV(BJw3;G6Jl#^}Y05nEgn~|JuKDHcZ#}o!b3F zw5-2gxJTMIjC7wlA81LsvBcrx*b)=XtTY!Tqt6w4(j=yvuXeO~$efG`4{do41dT8D3LkqvR?O%r8-TFZYFkotWM{Ja{TA zm7hh4+yOvFyZ~JvYMAcV+nSw>bHl%f`W1R}AW;%p;yb4S18%dtvyH;10Y!_wv=)YY z_fba0AHf=Bi4f*s0Ys)TT2d#)wWT&{_s<_7ok!Z!c-l&&#J6fe&NSoPl8CXXq28!Z zKMM{5pN5^f|}wx$l^!J7h2`-q2DlA!gqwOEe5aDT#GIjU+}#`%yB7fnD_ zvaz9os|Zz!&I>2K{Z>XsR7yv$;Ic}s{&Ctz+;xG6e{lA?OJD1U%=Yw0g|fm28Rw+Q zrNRQ75a3k>_$&2UDMEv*T01baLbUVV(1k&s(CSg+^|OykxD(Tdr3Y!ly48jcHE)lc zdf|XcfU`*^VDEXc32V^XX-=rc6L8r(g1(M5p!a;% z9Q!m2hsYR&X{?R02v0_(ZI~zdq$*JH`$O%fgO=qYBLnKr+Tcs<8U_x4da~c>(B~w* zqUG99Q3PfE{02RzVIMQDzgl@_olDOjVZf_<;rC+%gyWj-R}p-56p59E1pipattJxV zT+QLnWK=c1kvDgDQuY5-j;`A(CrM7kpXmB5`Fm%v1qI9lijBCB7?M*yqj6(A3Q-q2 zHi^pUlH8nh6Pl>KWK1W(|KVn1l4?0v^YZxnVr|Uo$f&fH;QUZ1y7HevMB?X1beE`QO2@mQYd*lq@Eb5~fm{d-OWoh!mfw67 z6qn1tU9U{Q%5-B{Qsj^K7(o(YlKrMoRSbdq*zbB$(B)dN>oTrC0^7}mFItlYU4-#% z);%!+SvP3Jfj`qe z3x}!|1WFx>ZVRBm4oTx?luA{7_|MM$P>~cxrLh&xt<||q*|#iz`aLuZVqy6MT_2sn zhOTu4M;cYJy|a_zi_wPFFjyqJ}l_9n0l3V1VfNdTvQCgq#*8|eJua`o_BgfEMHG1cqQI2T`^*ry2 z)g^>g2nchM7Hz6%g{Is_?o-Uj8BIrvThuuxfLGCEgIm+rxic)s-JoGzcN;$MA7|qK z6|1;#1X5r?qodkk)2IJhz!N9R<3LBZ6}M&+dHB%Hx*oXMT0AdlsWB~*D!3bdg$Mck zBmzzzS&uuQC5?f)DIHln%4zjIkkeU8T+v7nwv*X^)+hUEW<)Hj&DFfOZBj$h$|I74 z62~<``p`yV$VW|-wvzilc6Z8>s8X5x1*Qcw^L|R`etxz}94g-=AL9N2C5QKyS zp@NTt>OWjEx`he9=pDj0pWOr|B+^&o>Vra~6p(`jCEmfc55oK@E-=Q9Mp8v_ec&AJ z`0;l#OH1kt{D12xHN|Xz=n52>%hY~7?lb!tjeEj`#_BDu`RArZ8 z?NdTb#yv#C!N_PGxJ>^Ee1N2WKK%Zbn<+c_D#;8s4Sd#HPZjWd*?!)&IzJz(x|wdq z7q)VeYE-9IzIa@g$3EYjDmvn2cs1p^u-P(TPS^)nB>D$p)Rva=1DSX%E4d+p&UPBs z(sc!Wt5cqOu|(e&F*ou+pJ)R z`QNAo@V}0K1zCC%3+9J9QA_``a!;KCo;EstaGG*<6JBITlP*Ez*qE4|FMn5omaCyU zjiY>L-IvkFP*f*+*L9xeO8AG58pRWC3!(JV`e`w`6a?wfU(E%WP?c*<7E%<%Xt7)# zin;#Py#%=NaI9Sj=67LAeP)^VBq z(~$0mNhi2y7!V#wbiaO5T~s;&)1LTk3(0}m33LI78m4m?nF|RGRvP~f4apXCqI~b) zQCf<8rM8NtXn0l{M!lO5#j(>Ue&ZrP$iso5EK!|S5hLqcdg5^INK082BJMISz{Ct< zza_GST?qyQ6`lP@t@V|G^!H=va!cT!(UT@UtG_Z&?s&D;>y8zj`D~l2JlCD(z=G- z{(hF0p*@A=4CQEH`_x_ev^)suqq(kdA*Xw*7g|8z4^ zbffVeYerv7$rcCVWS-`!E=doTd)Ls}X|OKB$TGrxM(*!jX~eB5xiRYF@?1U zn;#ZG3`OY__fY#26Y31_i5Z1>mL-Xy3lgNVvC(61EVaO4?8TVxxOORKPq)foVR5m^ z`_?wu%P}?!nbA=4r3_@Le(K$Sj7JojoSYo1nXZUnCjVJFFWvz8QZdbD=P{zv3{z6Q z!JCkLd`h#;Q(|P`1d+gwF7TThqSJZ8c;fjMDL@=xO0T-!an z6RKM|jE`fZDWPkX*jL0l%gz`H-4^|IqpPH^#M3xhA?`XO5T!APDf8%ggvk&5EmDm_ z7DWU}1#qBPJ02O?Bxv?!-u;Kv{OTnGipJRg!07A<0pw(4g_;1;A7HABD5*@h`*77hg?a0)` zmUYdpnqN!t6zUMTt=4(>lo5QEw=MejvZJu_wJ%u2YlHvM&h0unUbr*5WO!!6#M?2$ zKNqW^bJX?aJ6iB`JGKRoFMHu7MOt{_7vHrr3s&N#lodlcdkiRkWncr*LMqXo&PwEGA(`lesK54L&yAzpHHaG!!Cz=Q&d0+%`7 z@5Mcr$9&E#5fo1>%e_P)M`1$@eZ=!@0)r@+ew=>|138x~AjHl@)@qg}BUKG5uKD)u8*t^46A~GWjqqz$W~-A2(E)|#7!iZ^B+Fzz zdBX3Q_kK4sA0}QI=xAB9lg(x3wNpUf_nE;&<+9tnAO9l@+xALHnnbNN@YWV$C*na- zMhRt5L~o~Etxv)~nIhRI;Z5`@!;8hkLGW2KV8)XS9uM7C4U=X~N9makPDnI$Om;0V zHmnc%1$=l8|NVr$HGA*267C+=os>!BcbokEd3UYVt&*map>v9#`3t9;2O_770-8A$ ziZ38D3c-$0Z*Z2#K=Kc%g3sW)e#!~j=OtN^fs(T~iK0<4Ai5oTA=Yqx-R9M1qs<9v z>2D6Sci{Mzz0}J#S!M7lX$I|ry1W;y3MuAbPGPDguEw-jPqtZ}03V;{(~{WH%b&eS z*SgKhfEim#Jrh2;{4had7Z#v?1di66)y|K^OR^5Z57A+N>sG1}lm43}BxS~yw5RdlLxFf>MUrd$5Lg~Z zt4$ZS*1;bmuuN1kJnopm-5ZlhX+{p4X*&nU6{sgL0^?wCLJ%8-qJ@4B^m&A{h$PMb zw!$P=%!g3h+`BQ%K3jTbK#lt+yD4e_&Ef?Ll14L$!aAglegZBZ4*hWTrV<@I; za@K$@84rO5A^t9xTVAeMCR&b{9o55ucC1Kg&SFzim1|U|+Aym2b6ZNnUu*K#1AZj7 z8w2=qlNj}bABfh2YC(Xk0^4WsQ4(4Qgr4X=d<#Si;wZ6)qa0!DEgcbOzHx?k7UJc- zCJMMD`bQP%2724cVe~8^Tn5NT5$O_3o{a3kOq4L`c*`}TIgE+z0y4*;4jnXSM${fQ zqgj0)Uk4|94tUJ~prpVeRT(QV%(C#VDe188uyl>^h@)EBsb_KQ>``3y5ON%5jQesb>Od z1wY_Vjz|Du-cwivE8_iVRH%DD(hqR;g1t@wJjq7`0mn6zsWDN5hCAw)XmoQp}O!+{n{jRab50enL*%&UO?h{+26c48;Tp;YHY15_BuxoNl zOHb==8P66$rk1EjXZj5YITZa8K;0#3 zJmEVpMlp1wRohStk}QiyAGAZEcPBIF+9l=3#WB~)|7MFr7+n~Bt*hG>)y8B3CywbV zxp&jWjjI}qD#hnITSs}swhyZ%E(aC!`5Rf%m2#C3ih}abVLyymG70bl?pcA;O|O9bv|T?Z78(SHi#iFT|THyNh_C}9)~@weD(JDI3a|*Q)Nx2o?gzgHqb|7q^UbIHI+ZL+3RGEwW`SR7VYsRk2V_v{ zUM4>PTJw!k3@tU(PkWFdLWFA+#9eqzgVY+9r(_y#X*#03+10U2a_++>pRL(FhipKx zXG^2v0eZLD1l$5p-9Zrw6f>}(x}BW_oiGgq8tJ?{~iSkMU$U)Z$Hbhzvk zUy-mvs!EYC&}t9g3-+cNh>l499055gMyILz>=9^yi!T3V;NJF3Ug__!cV|CuNX#h5 zsJAD2>|;g0`@teFus*@@B4to^Js!e^?*fDHx?u7`AdVKcCe-13dFpzL%iMsNuZ4|` zEhX(d+lPrd`}f<|{PppcgQA=3C$o^T(pu|;ycpsdTzt-cvxJ+LE)j1`+@Hz zLB=i4uzmw!2d>xVI_xRhBb-pe*Y)2oMnL}!0Q6gEMUe(2Qz8JRB{$eXw1W1u_(vKsXp6q&o#A}89=%Xfd%R!)SqumsG7T~B34I%$2iI3%*4Ly&wki7zD=G(>uPZDJ$&74S zjLOm*xtP;QUQ)v-FX^MOf{{Q5#E@5jX#HYKplQf?|$9` zfPa8TNpdP*P16>~JoT-Ais&tFVpDO^jhm=9@6ull{wFd5fY2^p| zNe>_)VR!Rksn-Ig71U9lFCfAWVJKBK9jUVWY-ca27Dn~aStgAsb&B{E0D{K1$SRM% zpmKjosWKq)C}m=Sr(wk;2OuD(7^W01?I=TG(`Z+=bPV|yp^paDJa+l=M$$QKvZ|6- zIqw6ygxMi`Rx)&2Y7HY+KllSN7nw!KUts7TfVdo>yTHe9dK(r*+GdB5tP`--kp~@$ zb){uU^Gc!t-5hVgYBECrP~+oih1yOfayF>naNVJ`T(E$I6^`|xojSQST#fvbd{?W7 zrZ^m)3C54+tA}hGa{j!N_~D%3Ay_5H1f5-NdWn(cJ|M|rs~i7ckd^kzYW;jZa9#}! z51X|bFc-&#i0I<73$cRMHzwN@5QXsHT?z1#tQe!_AC`Cpjz+W!Lhz$&pAGd6iV*#Z+~5^~S~qfIlQGp|&o_ z2caw!c%*#~ISH)we0h1H0yuL3Ev6w@r@~soSOk@#<<7Il;I=#s3JpsLRT`OZ8*nJ> zhi*!5Xu^g5T!AL`#S#K{rj~R;26<|uWI-A?3viVMGUfMY;kV*!l+ZG(AVHmel9xVq zu&pEjfX~01JU|E?KOB%zKHY><&X8`Yk%0^vWf9fzFvwWk?k~2WP#SB;L-L{a*`)%+ zT=n)3b*7s$OkJ=Oh+_ioMmOx?8U<2_e;ciH4=GsdCfPkHLvPO7&x5O+3$wwux{7OF zzC)Y`Ug%}U$n8E-t;(!IGDRuGjtn2W@S}B-HQ+)`3y2j*VH^m8)C$r*rD`8y6&t{T zVS?Og3F;7Lb?#b`hf2Q@<`lyOcS6#pg8&A)l%d3jV3KgvLb%O$E7{WTX zf<=3PQn;x$jUOn1bGf^_>uX^8VP?jh2Qz_{F@M^8q-9GV(7iC8fV=hR-Y;{uOnolN zelT|NRGn)8=LCi(orrOHo98SC+I;3}a`^HQ6m%v}bR6|dlC(U^qx+py3#53qfLJ=} zPVY$U56Kd(xtgo-2-=K{qpS4tq*@)|wyS}xBby2wy*f-FpPIw8DQqN~am0sn`OMNG zFCn!ESHk1AP&GPF4#%7g zLQ|-r5P~x{0n~n5))|pD6aw*$cswD|ps6`4UEJwq%=U zMA;fR&o&ttCU7QtDXxIk9s;G@N%#aicfPmVnqA({=YAsp>LppFDL>mnqE2z`ti!yr zN`oQCH!CAR^Q)A^KClKb(w_buGeCwiV_t2SE4SH{WU2r-p>--m^O**7M*UX|SPtA8# zPCh2)ogKl7F>JF3_>M}fzR&lUf*YRjEDdZcJkhS;-{p>l``M(SI*mpqfvXz20$B>e zM`(@j#?cxb6Q)gFO9+ng@ba0S)9^zWD7onnNl8=4YsF)a zGdNIoM(;*vmUTXj*?Xe$k{8y|c2rh)Q<|LOkjtp-h`#)7F*m-vydGeT8&LovHHGJrfvuSFaOsFYAs51*eI9Tp_A+ivA^lQ*yqKr^C#Ss(efn zC}&l`Ct`Ck{=h0Ioc?Lc%fJy6RgOvk+vq=~DVw^+-2z*m_61BaH_k9|hdKhO?zflo z;o;%m{!b1gZ@}aN=O4B7G&Ekf(`|`sM`(fUVD9XE0fg%V0;QUH$31Gb45@Zvo~tW8 zJ9!Ec3F{WHQmB5~ZS#h*wSvt6KLdEUvJFq5m9lB*M1Di|lp@{$mM|>WvFm21uFHLG z_{v6?Ua5LLDe>-b{{XNOfn3@9$uaZr(|R57J>`plsOa<5yYn1TucYymTZaeS7UKW3 zF0~yU*l`r(NU*6P)TmO-?Czt8km}%O9=MO*S%QJ@-s7`R<|#10TwxT=Q}d1O4Q{iP zxr)2X6aAS?h{(?dJOEw5F5|O@5WYzjBa4H>UQur_!m&|8iKPWy2|m&cLyC^Sl-q@9 zHP>d{AElc0y7>9M0G@~2t=?2T?3^cas*wXu_&|O|ZusMfLuzHqmg8zH5%*vPvL)Ls zNku!I7l+|^g1(6G$1n?LN2X#W7MchAig*6;b~Nfdlk>m)|MHd*6qd)f1M(tenJAg~IA zPVd$hc_zoV8nhZj`O?;S2eWLG*okQS_rX`dsQGW$!+!sq@DB6ONIWCG#N-e4kNWA#( zCDZs@wq?6p6ea7ZuQF-RV6G)Kb}PREP0V$jh{%mA9zXuASkf1nt}Cr9`di?~On(?P zv#aWyQ4d_suO_d1CLD$>J+DPS!o_!=5zfnL;pm#R(L=95rwB7l30{x}+-|$Y%H>wK z&r|8?xZOURJ+Kmqzp1wyDcNqB&?C-H5+=knSX^naEobKE@uidH53=ZCtNjO7&o4KY zN-SuML83 z?ez|TB9BkaUlMpbD@#aSez5>?oMC*>=s0X;(NtL3cYR4rtFZ(+U20_4EtYz&%IZHi zJAsV#=Mh?*5qyR)J~LD?hyfYjgSEQ%aCAk|R_Whys|a5=fWW^5jf(zNU3=1*v>4`* zy+J=_DOuHlkc*}qo_Y^--V__6)8S&H;{=vJxyF#cTCvGrUR_HgPW~YXNfLPl@O=`S z*uV~^o#A-9Jvr}w9wsGmT4t~K`7u8S?MD+{+&HPKbl1P%K2sxIPs}JX!-p~fC9Qc3 z$rSmv7!zv%&;@-vSY11g`)6!83kCxe4zih2|VOtvP zpyz)@vJ;a(ly;jHjuC9SiH8Z&4rUJXeKIpxJ@mhFgs%&!UJeDjAL6Wb$&krh5bG*Q z+xm?P^Oowb)AZ%Yl->IQ*X+5ijF~7DaC+$I=y1Z#!x?NMC6bEs*)7pUh@u5wepF7f z)^#>MpnjU!r62JoG0M#wAoJOxD8*4(%YL}s+S(HQ?TdGH@yF!t5eUAnk*KqieINVs z_s7s7=SS4Rx)B_7HZx8Mi-4EgB^uP;diM-D4f1~C88rnKPI^1meR%?xLy>otsNOR& znGaZ526lFK{zmgD=n&uh)>bc9*Q6Pj6i}VGL6MFmeX}7L1+S5VL0;3kU!XsmCVBaj zqU7u-zEXyA8#M;K=@KE`$?U(w-Xs8U5c6{sw~dTGcJ3H79-5NGI1?83Sr=Ez0SWkz z)Gx+9r;T%^s)Y~0V?IsE%P}BaWO;YvjQ90NPW&!g8lf$L7^_@6DjzC0@#xVmyXwpJ z`!_dzY#1jPnot_tkq4?}kWsq`?QBVK7mob6gNBafJI`_dbgu?AyVRd92}fPMWn#aT ze^CC*aU`pX2f3gue6OE1SGE0?V2PeW`1{$4?Pg4GLgh+l8z%ZO;O#Cz3cV8udcS<< z;dJU`neR+9Hz9$O`l_?|wkZ9TW}Kz0m?~(vGmW?+4vx7ID0cy>N(#X-z(D}kSq%f; z=fMGqeqSoHMB#|oa^*OgNm{pD2a}O_Qg09~t}V?a=XKgu$~z##553Dc<3xILvyll1 zcrS$87VK8aaQ#P5>b8t-nkj6Jshml&1JEb`{l=gaa!4++`kq!xpP5xD_>a~ov?LR#Q*idN5V7b5nM+%xu;N`~ zbS|xosG`U`TL4U%AAEuwxqot1qDO?ZZ{Fg)*F)JYlK3Er9*SA?0ATw|P{6$Rz8oY1 zR)yT~?P#)mI4m><+opGD>(u670~DA`r?gLh??p56G*oHQMR6Qn8SQG8mct2cXT#T`?(}33qOCi#l}v7_0$@di2q2hNY59|&^^#Z) zvQChb2=^3%_W1A1zg;qkkTspPH`V$`t8Q~C=<~$lDz6~9hpeAf{gbC7k~SQkxIZD( zYmr007=!~^*=X~yb9bM&e2SCPFv3D20DaBVV)eB0Q{SK)(NCmbuz=$shXJoCC6km4 zMsP&ek_iyFP+G&Pyp}r9Km~@FR7>_Qw)(loGD#z@=T)u9euuAAFxI74T60#9Ue$>e zU!iB4Q$=}pIQs%nsej-3C{DJkb^U_^J+(Au;5|^D0r(H3B>@WPUeGB@_PN6|g*8!%QdG#(u(E6yYn@Rq z_yLj40=@PsDU7_m2XF`Fj+?;CIpP7(3`zRzyu5Hz1T1Kr1l#*m`irIs6$ks4-$tZL z26$hi42BvZdw#5zUXFV7hdDoCek-CQP`{2XQY^Y_wf^u#_(XUR*9|QZW3WrPDfr?N zJmk~!k>%UpIYu57EA3UH5Rjr2}G!6hrLJ$S|Tnxfwe~e zE8V^oRdJKHyf&4~|uxVK`)$6tjs(5jd-z<@WotnlPut=k^z1`;s43!U~X?bn5 zTMCpasX?aUp+|V(+a@i30H2m~)SE#6&Y8cypG9tqlC0{R3R{8H<8J(J5&TuwsCAQ@ z_wFl}!6P6i&IpT%iD6JT2n4G}7jnq-WEo6x*e06mqw%%U-^T3n(0_oh7-Z@O4EcXv z^lNK^qqP;g0?V>uykV_qfhxhdo2+WA^k@$Xg!kRMcR)+daQT1*0V>}a1}Fg`>Y?5V zFK*~Ws1~WtXw4Ziy#6n)E<;=$pKNW>im>@})zJ+Pb1fg#_b`EYP<0NzY@DIKq@*lU0LB3M+|cS8?$k5GN6#($z0^#VPn#Dg~_kq~|pRngA|!RTXUP;O0_2`poYnNCCzDoau-2qw(jLGrHPy+2Y&a0b$>Yi-pN?24VU2-} z4FGJ&)qH_-;_hTwsyGa0b7RN-!JxjakqbR}8g@#xztXjnW51t`A@4bz}4TMvXWG^9l-6jvw1TQp;tZE6JdNFeBuTh+Jpt+ z!ruZxpr0iYW5i(RG2pDWUu#L^Pv^Gi&eZBZ;Nt9ZtEBZeNS-VNMT>Y)prZ4TH-nr0=41;T8H?fXC%$5MtQ7J%SSU}UqxVZFevu;YCk&xLv1!O~Vn+*Q`q%y; zu=M)Sx~9U$ZA`FJs@8dLfJwmpMl3LxAhJx5vO#*Mze6lQNZM&KPvyt5E}Pztso`&9Dt|;v~xhySYBIGIz{ZRWaNJ!oxDPavs$RTug-{u`*H%j zxbl}Uu&@Ac5U8X95Vi9w7e+CiPxMN1; zX8a4{Cu0sbp_zOq0JI93AOd+J9%zK3e35_}3gPO@p>{L!{ho5dVGy_-A4K5-!05G5 zo2X*-tn&u)EJXrX6g9gcr)GHw=H_($Z6mlfv`bTSiinMTs--T8nW(i$t+uH9L$uu= z!)lSSxO2F`mPrY1On1cj4^vw=96UEFoBQYdON`Zi0$S+m5n=A*;$42W{3>PsiYNqMjtrF!1@{7%wPbTqJ=szvEM_I(G*h>pe9$c#%x;2nY^v4gM z@C333$3T~DkawB|aF2-XDf<$b_iU1mI(odd%ktj%VYjE?Ar68ce~rb~cb(qZNLGP2 zjqY+sY^#>XBnL85Ok6zLc$a7kWEMg#DV5RZ-(Mh_Vq1)sZve-Y>$zh%B7AZhtx=s@ z=%A^Pe>UO82UCoT5GJHuSy@?NFzgY`6ju~cW-MFE3TqfYPChm-Ip&
        u%;ww~Hk z0r5!7f@yG2&h#%ELOHZyXpY;g#tHSriER#fsdK?`=8`h4TlE*8#HyP6mfIo3&idau zyfoergU z^qga6^!bgtnpUHU2^}%#qmt&>z<@@W#A3C82dVv9S?iu9bds;zRkuTK<&+4#BV_l% zl{ToTiBPVAY5UbCOo74e3Uvxi1UB+m@*U*^e$8dKK}H8})(*8@s5;90cI_8&J`j_C zljYXA2s=U8IBy%YxkDBmfcB3O+Gj&8Xj7h_cdsEPml|(80iWmh!$`d;OQU> zwjk?=giHK(SC1BLaGjx7X=xL$=D?C4Fn{|g9!Og;Oh63!K&v_`i%$c&(a~W2Z=-~S zUS{x`W-E)_GwN$$N+En2q%xm}Inz8@)|c_hG|5Vv;8&@73Xwbt$jP%ql;s+a@Oc2m zy;03&LW29~`1mp4)JWb9Ul-(GoEefh0|(KtgGT_N(3sixAbq&sHaQES^R0Etr^Y~v zk&&ZrXduim3(kvuG&Hf?^U+=hts8aDe?2VQu=6t6%ku(COrK7rc*SL&V~K=q0#7q*r|G&KFcZjCPvyFQu+O#)V&u=yRbD7!~7%u%Reko zUQ1?(DCIC}11m7odbD7US1%>?8CKgIS~R*7zTd1kQbwRCZBafH=~qR?G5b7@vD1hS z;u-hS{OW?tdZ{#m;KaRRCd#yb-e6f+5e@+jeNF>Ap}}K{qy=#(_XY70pDl8uCOEsg zY7A@wKi;esMrDSrR94*q2H(3gisR|9&;p4+A34*-Yog0}!K7d3a&I~``^n{GwdZ!F z^5RCE9gw7tM(bYv_hZH?9W0_;R?j!(W=c;5zyIy69}a~rEs)+{%Sz6X%E&Xzzs})K zv^BkSLbQecmk~d+CT8Pb5Pmpjt`hg)i=OLn9+28yk6*7Q1d5xE@i@X17^g zVxKdOIi$Xd3?ib05At2xjmA3OYh^ZNf8RJl7fjz0_pq_~q&L&j&J!T$*)&NMf^ozC zpX@8Wr&?Q5yEB3=u3#*0I2}FHiz)Kh-|6Y;<6~Pl0`+`)Fn+N+z>qMgLK{}o-qhN{ zP&~4xzX~t%%4l?64e-D<81OKkr4EbD-1$?#vd6OE;V`wSt8%%R?pv#G!b~U*U;kXt zC*}#Yx~27z&tR^21O^EE!EcQ$PWJ$Wn-m<`N`ve`*^3sNB6`%a?&COfAJXq^;| z-QJaN`b~72HS_TYhkBd8$@LhDbNZGLzff-c?tES<`2rof@OE_qnHAQ=XwA69kS?zI zAZD{h`C6A>yNu6&eY-Con;oNEP`H^eLbLIEMR?S{BFA>HO;?&YUW@5{^kd_C)8&a* zaZGM&wNMLxA|IKBG3wXRryH}a-?)0Oud9EGb0|Y#X}YVd1Cqdq#*hvNcF-OIQ}7&` z48AoDDu6HOsH>wL{@L0ZH5gr3e1{!FM#Gj2Sr=YvNz(#bQ!z7=&~h4L7n?g&UzU@~ z?xj%Q*SO=uw_^>V+G~&!6Z0L}$mI_QrUPEX>&={M)oTA@S}nj1d)wTUz+eIUN|^OS z+lrQoZAdJGVS6Q-gj?CexYZ^JWXKKw@XAfnVHchlw2+_6gy?J&b?vE?H9meW!93{J zM;Vi#gIN}qt)_5wm+=c>vpMQ&Obg{mi+Fl^vK~MJjk!Q#d!#MxB=k(CKIW=BawZG( zFhT{aYI|AO|Kj9px#!XDLblc4a=ZO(v-q!e^qq41-Wl9|45L0wGCQ!J#m4m#l~Iq$ z2nHlQbaZs~_0F_pXBU^msAp3m^kzHxyjtUQVY`O1ia+QL25m#ZoVFMP{RDAa&~ELe zpv$nlSVyel4bUot=LK`TP@n9bNFXO6) zmv>ot-7D$mfA8$r*V9)R!@Kq54P>Q}lc0`m_-X-ShrMY0h+jWyF2C&MlKl9EB+m>Qk zlOVUZ{#L&^5!8|CWa4aC+wXokv_3|BeShaF+ za(*`-Sw5O(0HKwXmI5R%#-T0Xy9o~u-%OG;p&R(EMxP%RF5D^>g-2n7C8NsN(iM75 zfnxGdS^m5cQ2uZ=(HIEx%o1cDL;DEqG+FEPX?AE|7YCsMJGjIF*f-GS0J?vuFShcf zBm&mXE(5`n3`&JGORPXIQ?Tbw6W=v5@p|zf89Qjcpipp=6~8hL{((#%6lcl4D8tg);-}_RWW1EtAxVI zG3n{+m+MpGq=%#XBUc=7fpY3;!@S&~aRttk6BCgXp1__^IFds&Mrh2f#$f@lqc5y4 zFuViEXj#|oCbe;{U#rg5Rmhk}$>v`R`i^H}i6-OWl}je7Nr(Bn4*{>$0J5FFF!Vu6 zqf4;63yuj2gYCioeu;G{3c=XX)5TUu$6?a`vS%XqOV%;N328+a^Gxx`*gC)Pa!G)| zTwP)|q6~gs1_0cRAp}Sz214*g5eaJ4EI~PAfX{OZNC+YkLK?h7rkGj!D((esNkN@4 z`O0KB!4M+E7h(*Amx}^xJ)r0Ww4Pc4&J!covbL@0uB@!A6nzp)>cGFhU&B6Mn0AV7R3Kr(d481d z@sDTm>drmA@BKG54x}$!rm{?hxG2Numuf{Z41fE2fZK`xYsLUkLA@MPQ{8_N7gkZm z$6BS{O7C_Ze$nffJpSp0YXx<};=%JJO@FtW%psk*^#JX@teE6c_JrX&{xlHqax&R?Z z7nfRil0%V%(7?lDS4fxza76j|_)rrg=Etm>1qR6;LO}}Y27glKHv>{dPjK#KUP?Ah zSPSOga|aVLeD@8v3NodB<_4+;9PzFz-&|8_Y{1t`{6S}IH$ zo!mbN#dvm4`osF2Z0e3|G2V;{c8w#biT$0Sj>{7hpUgOreiVGblVj>>2gDD9Z!?(~ z?zu{RP-3B(f9m+u)VG{%Ak06;N+s@_pG8V9Rb*4n`?i{bLh`s!MaDAYueLQ`70ySD zF+X{1C3QA~F)4*8!)N~K(>^d?05?>fYqWTak~;@#5DRidJzp`y~pAP(&4kjs=En_D4kBZ9rh7KEn$t#riWef%~N%7HnLwv7DH z3_^qOXbk+ql75oIw&2iUg<{$};jO%!%XCf?N-C;%3rN{SqvbSt)*a(W0=)`MJFU!* zOp4F&cwu++P|8?D5+;lkOrm$6roO)q0G7`I`IE`1DMCLcOz1l~CY)1qD=WmW;#)Ke z>KVv6W5HhXYRbz}iQE|LZA13>e!lb?qr_PrD?|#d!*bMUbka9lw$U36ZEY-yGM1(@ z$n#8X;h}Jep#{I=|0C5($Ji?a`$Xt6 zNk|M14vN-+J_RBY_xJa4NvhSVa7^Bn>m>9O)6pff3iKdEfT?ggF6_7BE#afVMmSNZ zm5$#R)P&&q6ij16Hp_EiBFa@3wR2`>=IYg}OG`^#%0Sn>By_QAy12L~r(TTm=g*(N zdGki3oN!Vum{1NWH6jU7OD1o@r^@>-%Yq}XSAwsp8a8&R`p{hDSkd_;)HM*)Jhk#k z9#Y|iFhrCX<2ywFPE1TpPfsr_EXXpD>t6KvI6bS_~pp_NgBMt$%9ir$@DDH;f*gi$)a}FJ4?-T@@{KWMri1GtmTewdmjdxSC@8-qKSmX<^XE&3>%dd7Nk zJslhz{Q2jfPo6vx)l@hMgHV8x-GC+`w>1MvLh#e*qI%;C8`O6)f;wk@`g@e_Jz+@5 z(g!^t$wc9Gzj_G3$;n9(fWknLgh)^8CG;i<1wu2S;@aBU)2B~2H#e)*Duf|rpx{3- z&~=~U(b$KiC2d7Rzo4ro2aSBJoqWy@foA!rvPfvEAN-C?D1q07x%7YPVhG60p`oEl zr7|@&wYa#ru&{uUvb~3%B_VKpeEj_R^S}T8`^AeFZ{NNZbre(t4-vI5;@ihLck``h|pm$UtG^ zqeqXPJbALcy)E0qDld4QQL?zC(>Qa! zYqm|KLF=EvUi2ms2yK%!f%Jt68^_1TXJ= zQs>s7cLrZENj4_!#wl5+gB+S{K{K-H2istZnKAIVVd4#Z(a*is(vPtRoed)l2XLpTS&E*&ZDo75Y^wmHGMk%a<<;C;ORx|6GESgaF7UvhL~Ar;v)f zySo@yN|DG~VK9xHB=`8}tEQ&&VlWjhi-fgw|NpE*Xxs&Abj~aRsMD;YQ1f}+cTiK2 z5kI_$p1i}u!@|JX*;!$r&<_FnqAz<#pd_IMT%+qTL@J8iNF7xkkzP|vGHgImZ6fMN zM*NK=^j$cHgobiD;1P7L*7bAh+!vpC&TFznkBiGFdcenYtdGU-6o9OJ9~l`LA0MBd zo)-Fv3W|4&zRn$ll7wNP3<^5x#>R#SM$|*0b&c=}f>C==H4h;kU95KEb>Dg85|QXr zk)_6dP%#*I1PMtS^A(-Q0`QdqHS|h5U^EHII?Y26Nyj7*=wr%@yn-fZ6B84nd5YnO z_ynAyB{WJB<^ejYFzVT}XTrwq?QJm^4-O8Xl+t;l-KTWn=vbl-`UgpSZf;HlAR>W9U(C)$Ny3I8 z^blTwiaR?y`}_MsMKqYAhlXs_6~lh2O|XF2cvD?Ez-Uy^S}5*sNNB7+O+uf6QM!Aw zpZ2Lsjp%gIN!XZ%D^}mo(2zVr`4r8wKSK^DMM=VcAR1vQ>RXI1+i{vD3|%PTt|gvJlTfEt>$Fdj&?RZv zOgdwgtvm{^#Ogzr7q(-9KZb{|3no6tQ_%Hp%dd4;quy4ae@VEzyNia1APg3qc8&aV zf|AJ=rdz2~D@kOa(C_f@@Hhs_qbL}4F1QYFm^4%)2A!7?p2b6#cBmS+r%^jyBGaPK zz$1z~)BF>>LeI}LXU<^!7>LIngmk@fR*w#5a3?`{?)*SJ%uF`qQDkY+i=r6xDG5cB z6wWY3+j$iaWCcm6Get=ltSTL`&?(?-Mr^A1As)3D)S}X;qdDH1C_R9YM;A2tqKtI2 z0k7dFsPP8rS9mPkAM~+zF&jm?7QH~)ioPQux?NEJ)Fh0O zDNb7K=oo`>a=MKAdirUgml_7@gc~CX@z;3Ec>xTU&Z^;R*8A^5%BN0zaVqXNkTlrU zZcm*8%AQMl`U=R2zh?CyInQ}@x|lFZ7uxH~p><;hLPW;)l~nAnjG_1M-;3p@nMP@Y zb>Is_W3ZXXyGx5r`aeN1*gsg=##;j&*w}>Hm*YYtbQ#_nUj@}014(1gTDtALRz9^C z)X&KV>!=8jOE_=3pb2{(#dCPHbfw(ZHeuxDB;Y{`b!3j zJ3+TlP_%0AsG+Xl%$YNlN~P#E(r)w}2_eu(#nib_t`0-RV3i=?)5dQl%un)X^(%g) zUMrbK7sU$JkrEe5;PzR;PdZOh8SOxSjJet*p$l~rHK%dFtGb|+*1jCQ{}6y$N9`Z< z9SMQD_MR+-w{PEKkdvQf=Dane$!TO@F!)ZbOiNnIM&}u>TAUiLx_TsZ>HCY3U`ClV zI{847%NBX%g;`2(r2L$biWYwL)mIY}6Q47>d-!DTAM_~+8v)n7tRYlUxKK#IshVAv z7H&rFX*wCHtJ46m!4=HEfC8+4L@($_wHq{4^p#s*Sn1EZkl)qk9-Tc=+H=FV1_uYn z$H(>I_btuxQIfDB7~Ri#l91c7{!nS*Jh^8yOr!L25&M$RB_sr4g1I$R9l0m|9{M{I zd2xJ0R~{5TbN1}nv9Ynw^(KvH3FHz%lq75kPI^{lvF`5f3Uwnjq7-SF{)!h$?8VTW zi4qs?$jHd(=&1gdyk4zV<@fOcyePjczkiDTEc(>;AW9N;5X`m6U-4NNF6&z-`tcI& zL`lLDB??C0xr0lT=roihEK#Chlq4)sqF|IHEK#Ch{2yDykWdKcYZXhx;AWdO;ATlsAGaxZFIx;poF)$!2FflMNFqEPc00007bV*G` z2j>MA4IBuXb3l~<000SaNLh0L01FZT01FZU(%pXi00004XF*Lt006O%3;baP0007Q zP)t-s|NsB`)ynpnE`^rejLVKx8${{8aq z^qhd~UNHUd>HhZf{ORKS=HBpqS@^)M{qXDi-O=uHO7^dq|NQ#>?B)C3)a`9U_OqS- z`uOvcbnRj{`Np*T=;8LYp6+Ei^O14<Qf`{cvAGEi22jT`qIVmi)!|; zne(EE`N_HY%DVNclJusG?|@wMmU!}uYXAKF@{Miz(ZbwMBH&OWk24m%L>;t29NA4F zkTVv5; zulw1~{P_31hjxoXD~LZRw|#5#=Hlzv(zADDi9aZiNHM~Xe)Ob^@Z#O9ZCRmSN#@VT z-@>?W)H>g2_gg{x~<&!(FD-PFvXlfR35*0H7a zt(KHbHiSAOBqscBVrEEJ<*Ox3TW&ZU>1TSj~_ z8Ln_$&GqvN5B z%19sAbyn}gw7f$d-H~|iZ%62_oz;9`?O!s=WkkP8B<#AX`q#?ymwE4XP3>Sa>|HVK zYC-P5u=v{0^OAD;&A;7nOw~*v?XjKDa8m5Ntntaa)_-I4&%)S-YuAHm#8xuxy{`M? z+Vwy8m;e9(0d!JMQvg8b*k%9#17}G@K~#9!otEcclQ9s-!z%C;sMf+Sse+t1sbQT+6bg zn7QxTy7~rwDWdT$MU4^sYD0ZpEeDwSdr@UH8q?82bqxyi%UAdfc>M-_)>NbQbZA(` z-w~9jWkvxkuK?h|!@|l(kL}!(r-Gc^ZD^j07`I>CBb3=wcnaB_J|lu>&%t5g`3nFP;Gl*(4jahi0&pPj;Gz7(_NX2? zdhGZK_?^q(pq5JjJD6i}R39uk1Zd_`F2e>gOlH=8in22KRSwMHQjSaV)HECDTo6o4 z<&zwU<=4F=7mN+!MV5g(cJA7}XK!5czGNz>#9m_(+_H6BLgI#v+Y{ocpjvpWkU}M1 z#UpUjW^yu=x7FeAajXPfy=HCPy7iLDSo}rS3){!Ij!leUswYc8v@0%}ECKeS6B)Gv z(@JQ%3L8g7N{fzzGPewZ%V`T(+Y&q)kZ4*crF`)c2riu~npJ5ho0f|_+ zh**&{4`jPCYc>(gIoP&3d81?BbB6R7W`^7LlX3QzY}$0n2@0u3YstjePM$(N`>C>c z;zDibL Date: Thu, 23 Jan 2025 06:49:30 -0500 Subject: [PATCH 063/119] Undo hardcoding --- scenarios/AksOpenAiTerraform/terraform/variables.tf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 9bc0a2840..594d5e6d3 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -10,11 +10,9 @@ variable "location" { variable "openai_subdomain" { type = string - default = "magic8ball-test465544" + default = "magic8ball" } -# -test465544 - variable "kubernetes_version" { type = string default = "1.30.7" From 66527aa918278e3ddab85b320039fc6714ae801b Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 06:50:31 -0500 Subject: [PATCH 064/119] Move --- scenarios/AksOpenAiTerraform/README.md | 2 +- .../{terraform => scripts}/register-preview-features.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scenarios/AksOpenAiTerraform/{terraform => scripts}/register-preview-features.sh (100%) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index f4aec438f..f28d9b87f 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -13,7 +13,7 @@ ms.custom: innovation-engine, linux-related-content Run commands below to set up AKS extensions for Azure. ```bash -./terraform/register-preview-features.sh +./scripts/register-preview-features.sh ``` ## Set up Subscription ID to authenticate for Terraform diff --git a/scenarios/AksOpenAiTerraform/terraform/register-preview-features.sh b/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/terraform/register-preview-features.sh rename to scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh From 66d27b97ef0c2c2379ac873c572dc6c08f5c420b Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 12:35:25 -0500 Subject: [PATCH 065/119] Temporarily hardcode --- scenarios/AksOpenAiTerraform/terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 594d5e6d3..2d89304ed 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -10,7 +10,7 @@ variable "location" { variable "openai_subdomain" { type = string - default = "magic8ball" + default = "magic8ball-test465544" } variable "kubernetes_version" { From c2b1cab03af3f79690dab4580aa493603269f78e Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 23 Jan 2025 14:44:50 -0500 Subject: [PATCH 066/119] Format --- scenarios/AksOpenAiTerraform/terraform/main.tf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 14127733f..1520fe770 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -198,8 +198,8 @@ module "acr_private_dns_zone" { name = "privatelink.azurecr.io" subresource_name = "account" private_connection_resource_id = module.openai.id - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids["VmSubnet"] + virtual_network_id = module.virtual_network.id + subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } module "openai_private_dns_zone" { @@ -210,8 +210,8 @@ module "openai_private_dns_zone" { name = "privatelink.openai.azure.com" subresource_name = "registry" private_connection_resource_id = module.container_registry.id - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids["VmSubnet"] + virtual_network_id = module.virtual_network.id + subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } module "key_vault_private_dns_zone" { @@ -222,8 +222,8 @@ module "key_vault_private_dns_zone" { name = "privatelink.vaultcore.azure.net" subresource_name = "vault" private_connection_resource_id = module.key_vault.id - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids["VmSubnet"] + virtual_network_id = module.virtual_network.id + subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } module "blob_private_dns_zone" { @@ -234,8 +234,8 @@ module "blob_private_dns_zone" { name = "privatelink.blob.core.windows.net" subresource_name = "blob" private_connection_resource_id = module.storage_account.id - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids["VmSubnet"] + virtual_network_id = module.virtual_network.id + subnet_id = module.virtual_network.subnet_ids["VmSubnet"] } ############################################################################### From 356c53d54014149e19c304b8fde83231648073e1 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 20 Feb 2025 14:53:19 -0500 Subject: [PATCH 067/119] terraform fixes --- scenarios/AksOpenAiTerraform/terraform/main.tf | 2 +- scenarios/AksOpenAiTerraform/terraform/outputs.tf | 3 +++ scenarios/AksOpenAiTerraform/terraform/variables.tf | 7 +------ 3 files changed, 5 insertions(+), 7 deletions(-) create mode 100644 scenarios/AksOpenAiTerraform/terraform/outputs.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 1520fe770..1591d2f7a 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -57,7 +57,7 @@ module "openai" { } } ] - custom_subdomain_name = var.openai_subdomain + custom_subdomain_name = "magic8ball-${local.random_id}" log_analytics_workspace_id = module.log_analytics_workspace.id } diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf new file mode 100644 index 000000000..c9c982639 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/outputs.tf @@ -0,0 +1,3 @@ +output "acr_url" { + value = module.container_registry.name +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 2d89304ed..c4ed6e238 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -5,12 +5,7 @@ variable "resource_group_name_prefix" { variable "location" { type = string - default = "westus3" -} - -variable "openai_subdomain" { - type = string - default = "magic8ball-test465544" + default = "westus" } variable "kubernetes_version" { From 48043ea94415f4658c1b3502d19c5e809c54d55d Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 20 Feb 2025 14:54:57 -0500 Subject: [PATCH 068/119] Fix readme --- scenarios/AksOpenAiTerraform/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index f28d9b87f..d81e304e9 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -1,9 +1,9 @@ --- -title: How to deploy and run an Azure OpenAI ChatGPT application on AKS via Terraform +title: Deploy and run an Azure OpenAI ChatGPT application on AKS via Terraform description: This article shows how to deploy an AKS cluster and Azure OpenAI Service via Terraform and how to deploy a ChatGPT-like application in Python. ms.topic: quickstart ms.date: 09/06/2024 -author: aamini7 +author: aamini7 ms.author: ariaamini ms.custom: innovation-engine, linux-related-content --- @@ -21,7 +21,7 @@ Run commands below to set up AKS extensions for Azure. Terraform uses the ARM_SUBSCRIPTION_ID environment variable to authenticate while using CLI. ```bash -export ARM_SUBSCRIPTION_ID="0c8875c7-e423-4caa-827a-1f0350bd8dd3" +export ARM_SUBSCRIPTION_ID=$SUBSCRIPTION_ID ``` ## Init Terraform From 91d85a762a8d118e39e76723f3936df04534b734 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 24 Feb 2025 15:25:04 -0500 Subject: [PATCH 069/119] Fix key vault --- .../AksOpenAiTerraform/terraform/modules/key_vault/main.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf index 94c357af1..a23b4448f 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf @@ -12,10 +12,6 @@ resource "azurerm_key_vault" "key_vault" { purge_protection_enabled = false soft_delete_retention_days = 30 - timeouts { - delete = "60m" - } - network_acls { bypass = "AzureServices" default_action = "Allow" From 6e5544bb7d2fffae15f61e1897a7aa4c200db2ad Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 24 Feb 2025 17:56:32 -0500 Subject: [PATCH 070/119] Fixes --- .../scripts/01-push-app-image.sh | 4 ++++ .../AksOpenAiTerraform/terraform/main.tf | 4 ++-- .../terraform/modules/aks/main.tf | 10 ++++---- .../terraform/modules/nat_gateway/main.tf | 24 +++++++++---------- .../AksOpenAiTerraform/terraform/variables.tf | 2 +- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/scripts/01-push-app-image.sh b/scenarios/AksOpenAiTerraform/scripts/01-push-app-image.sh index c0164b0b3..6d194c4d6 100644 --- a/scenarios/AksOpenAiTerraform/scripts/01-push-app-image.sh +++ b/scenarios/AksOpenAiTerraform/scripts/01-push-app-image.sh @@ -1,5 +1,9 @@ #!/bin/bash +cd terraform + +ACR_NAME=$(terraform output acr_url) + # Login az acr login --name $ACR_NAME ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 1591d2f7a..01c19de7b 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -72,8 +72,8 @@ module "aks_cluster" { kubernetes_version = var.kubernetes_version sku_tier = "Free" - system_node_pool_vm_size = "Standard_D8ds_v5" - user_node_pool_vm_size = "Standard_D8ds_v5" + system_node_pool_vm_size = "Standard_DS2_v2" + user_node_pool_vm_size = "Standard_DS2_v2" system_node_pool_subnet_id = module.virtual_network.subnet_ids["SystemSubnet"] user_node_pool_subnet_id = module.virtual_network.subnet_ids["UserSubnet"] diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index b2b77ecb4..fc527fb9e 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -1,3 +1,7 @@ +locals { + zones = ["2", "3"] +} + resource "azurerm_user_assigned_identity" "aks_identity" { name = "${var.name}Identity" resource_group_name = var.resource_group_name @@ -27,9 +31,8 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { vm_size = var.system_node_pool_vm_size vnet_subnet_id = var.system_node_pool_subnet_id pod_subnet_id = var.pod_subnet_id - zones = ["1", "2", "3"] + zones = local.zones max_pods = 50 - os_disk_type = "Ephemeral" upgrade_settings { drain_timeout_in_minutes = 0 @@ -75,12 +78,11 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { name = "user" vm_size = var.user_node_pool_vm_size mode = "User" - zones = ["1", "2", "3"] + zones = local.zones vnet_subnet_id = var.user_node_pool_subnet_id pod_subnet_id = var.pod_subnet_id orchestrator_version = var.kubernetes_version max_pods = 50 - os_disk_type = "Ephemeral" os_type = "Linux" priority = "Regular" } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf index dc8da73a6..bb16759a9 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf @@ -2,15 +2,7 @@ locals { zones = ["1"] } -resource "azurerm_public_ip" "nat_gategay_public_ip" { - name = "${var.name}PublicIp" - location = var.location - resource_group_name = var.resource_group_name - allocation_method = "Static" - zones = local.zones -} - -resource "azurerm_nat_gateway" "nat_gateway" { +resource "azurerm_nat_gateway" "this" { name = var.name location = var.location resource_group_name = var.resource_group_name @@ -18,13 +10,21 @@ resource "azurerm_nat_gateway" "nat_gateway" { zones = local.zones } +resource "azurerm_public_ip" "nat_gateway" { + name = "${var.name}PublicIp" + location = var.location + resource_group_name = var.resource_group_name + allocation_method = "Static" + zones = local.zones +} + resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_association" { - nat_gateway_id = azurerm_nat_gateway.nat_gateway.id - public_ip_address_id = azurerm_public_ip.nat_gategay_public_ip.id + nat_gateway_id = azurerm_nat_gateway.this.id + public_ip_address_id = azurerm_public_ip.nat_gateway.id } resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { for_each = var.subnet_ids subnet_id = each.value - nat_gateway_id = azurerm_nat_gateway.nat_gateway.id + nat_gateway_id = azurerm_nat_gateway.this.id } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index c4ed6e238..e9809190f 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -5,7 +5,7 @@ variable "resource_group_name_prefix" { variable "location" { type = string - default = "westus" + default = "westus3" } variable "kubernetes_version" { From fcf07ed77421808a1777f4631095e8d3c64e6660 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 24 Feb 2025 20:16:19 -0500 Subject: [PATCH 071/119] Rename --- .../terraform/modules/nat_gateway/main.tf | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf index bb16759a9..1cb1cae21 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf @@ -1,13 +1,7 @@ -locals { - zones = ["1"] -} - resource "azurerm_nat_gateway" "this" { name = var.name location = var.location resource_group_name = var.resource_group_name - idle_timeout_in_minutes = 4 - zones = local.zones } resource "azurerm_public_ip" "nat_gateway" { @@ -15,7 +9,6 @@ resource "azurerm_public_ip" "nat_gateway" { location = var.location resource_group_name = var.resource_group_name allocation_method = "Static" - zones = local.zones } resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_association" { @@ -23,7 +16,7 @@ resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_asso public_ip_address_id = azurerm_public_ip.nat_gateway.id } -resource "azurerm_subnet_nat_gateway_association" "nat-avd-sessionhosts" { +resource "azurerm_subnet_nat_gateway_association" "gateway_association" { for_each = var.subnet_ids subnet_id = each.value nat_gateway_id = azurerm_nat_gateway.this.id From 1bf3cf8b10798a29a059db5dd86e7c8ed08efa57 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 24 Feb 2025 20:16:32 -0500 Subject: [PATCH 072/119] Change SKU --- scenarios/AksOpenAiTerraform/terraform/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 01c19de7b..4a9dd027d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -71,7 +71,7 @@ module "aks_cluster" { tenant_id = local.tenant_id kubernetes_version = var.kubernetes_version - sku_tier = "Free" + sku_tier = "Standard" system_node_pool_vm_size = "Standard_DS2_v2" user_node_pool_vm_size = "Standard_DS2_v2" From ded72843e88a2a21c0a650c38ce84e1527314cf1 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 24 Feb 2025 20:52:44 -0500 Subject: [PATCH 073/119] make inactive --- scenarios/metadata.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 28dd75aae..de097c05e 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -930,16 +930,17 @@ "configurations": { "permissions": [] } - "configurations": {} }, { - "status": "active", + "status": "inactive", "key": "AksOpenAiTerraform/README.md", "title": "How to deploy and run an Azure OpenAI ChatGPT application on AKS via Terraform", "description": "This article shows how to deploy an AKS cluster and Azure OpenAI Service via Terraform and how to deploy a ChatGPT-like application in Python.", "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/refs/heads/test_terraform/scenarios/AksOpenAiTerraform/README.md", "documentationUrl": "", - "configurations": {} + "configurations": { + "permissions": [] + } } ] From fc066b69a6113c7d4df355751562b9a9e4cb2928 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 24 Feb 2025 20:54:18 -0500 Subject: [PATCH 074/119] Rename --- scenarios/metadata.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scenarios/metadata.json b/scenarios/metadata.json index de097c05e..d2b8cad0d 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -936,9 +936,10 @@ "key": "AksOpenAiTerraform/README.md", "title": "How to deploy and run an Azure OpenAI ChatGPT application on AKS via Terraform", "description": "This article shows how to deploy an AKS cluster and Azure OpenAI Service via Terraform and how to deploy a ChatGPT-like application in Python.", - "stackDetails": "", - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/refs/heads/test_terraform/scenarios/AksOpenAiTerraform/README.md", + "stackDetails": [], + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/test_terraform/scenarios/AksOpenAiTerraform/README.md", "documentationUrl": "", + "nextSteps": [], "configurations": { "permissions": [] } From d8c99d6d0232750b012f37dbc83f61e65f8e21f6 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 24 Feb 2025 22:27:07 -0500 Subject: [PATCH 075/119] WIP --- .../scripts/00-variables.sh | 18 ++ .../scripts/01-push-app-image.sh | 12 - .../04-create-nginx-ingress-controller.sh | 36 --- .../scripts/05-install-cert-manager.sh | 31 --- .../scripts/06-create-cluster-issuer.sh | 16 -- .../07-create-workload-managed-identity.sh | 41 +--- .../scripts/08-create-service-account.sh | 122 +++------- .../scripts/09-deploy-app.sh | 21 +- .../scripts/10-create-ingress.sh | 9 - .../scripts/11-configure-dns.sh | 15 +- .../scripts/build-app-image.sh | 10 + .../install-nginx-via-helm-and-create-sa.sh | 219 ++---------------- .../scripts/manifests/cluster-issuer.yml | 2 +- .../scripts/manifests/serviceAccount.yml | 10 + .../scripts/register-preview-features.sh | 46 ---- .../AksOpenAiTerraform/terraform/outputs.tf | 4 + 16 files changed, 120 insertions(+), 492 deletions(-) create mode 100644 scenarios/AksOpenAiTerraform/scripts/00-variables.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/01-push-app-image.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/build-app-image.sh create mode 100644 scenarios/AksOpenAiTerraform/scripts/manifests/serviceAccount.yml diff --git a/scenarios/AksOpenAiTerraform/scripts/00-variables.sh b/scenarios/AksOpenAiTerraform/scripts/00-variables.sh new file mode 100644 index 000000000..dba29b422 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/00-variables.sh @@ -0,0 +1,18 @@ +RESOURCE_GROUP=$($(terraform output resource_group_name)) +LOCATION="westus3" +SUBSCRIPTION_ID=$(az account show --query id --output tsv) +TENANT_ID=$(az account show --query tenantId --output tsv) + +email="paolos@microsoft.com" + +# AKS Cluster +aksResourceGroupName="CoralRG" + +# Sample Application +namespace="magic8ball" +serviceAccountName="magic8ball-sa" + +deploymentTemplate="deployment.yml" +serviceTemplate="service.yml" +configMapTemplate="configMap.yml" +secretTemplate="secret.yml" \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/01-push-app-image.sh b/scenarios/AksOpenAiTerraform/scripts/01-push-app-image.sh deleted file mode 100644 index 6d194c4d6..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/01-push-app-image.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -cd terraform - -ACR_NAME=$(terraform output acr_url) - -# Login -az acr login --name $ACR_NAME -ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) - -# Build + Push -docker build -t $ACR_URL/$IMAGE ./app --push diff --git a/scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh b/scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh deleted file mode 100644 index f059c37ea..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/04-create-nginx-ingress-controller.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Use Helm to deploy an NGINX ingress controller -result=$(helm list -n $nginxNamespace | grep $nginxReleaseName | awk '{print $1}') - -if [[ -n $result ]]; then - echo "[$nginxReleaseName] ingress controller already exists in the [$nginxNamespace] namespace" -else - # Check if the ingress-nginx repository is not already added - result=$(helm repo list | grep $nginxRepoName | awk '{print $1}') - - if [[ -n $result ]]; then - echo "[$nginxRepoName] Helm repo already exists" - else - # Add the ingress-nginx repository - echo "Adding [$nginxRepoName] Helm repo..." - helm repo add $nginxRepoName $nginxRepoUrl - fi - - # Update your local Helm chart repository cache - echo 'Updating Helm repos...' - helm repo update - - # Deploy NGINX ingress controller - echo "Deploying [$nginxReleaseName] NGINX ingress controller to the [$nginxNamespace] namespace..." - helm install $nginxReleaseName $nginxRepoName/$nginxChartName \ - --create-namespace \ - --namespace $nginxNamespace \ - --set controller.nodeSelector."kubernetes\.io/os"=linux \ - --set controller.replicaCount=$replicaCount \ - --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ - --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz -fi - -# Get values -helm get values $nginxReleaseName --namespace $nginxNamespace diff --git a/scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh b/scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh deleted file mode 100644 index 3fee03e52..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/05-install-cert-manager.sh +++ /dev/null @@ -1,31 +0,0 @@ -#/bin/bash - -# Check if the ingress-nginx repository is not already added -result=$(helm repo list | grep $cmRepoName | awk '{print $1}') - -if [[ -n $result ]]; then - echo "[$cmRepoName] Helm repo already exists" -else - # Add the Jetstack Helm repository - echo "Adding [$cmRepoName] Helm repo..." - helm repo add $cmRepoName $cmRepoUrl -fi - -# Update your local Helm chart repository cache -echo 'Updating Helm repos...' -helm repo update - -# Install cert-manager Helm chart -result=$(helm list -n $cmNamespace | grep $cmReleaseName | awk '{print $1}') - -if [[ -n $result ]]; then - echo "[$cmReleaseName] cert-manager already exists in the $cmNamespace namespace" -else - # Install the cert-manager Helm chart - echo "Deploying [$cmReleaseName] cert-manager to the $cmNamespace namespace..." - helm install $cmReleaseName $cmRepoName/$cmChartName \ - --create-namespace \ - --namespace $cmNamespace \ - --set installCRDs=true \ - --set nodeSelector."kubernetes\.io/os"=linux -fi diff --git a/scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh b/scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh deleted file mode 100644 index 9ab805a54..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/06-create-cluster-issuer.sh +++ /dev/null @@ -1,16 +0,0 @@ -#/bin/bash - -# Check if the cluster issuer already exists -result=$(kubectl get ClusterIssuer -o json | jq -r '.items[].metadata.name | select(. == "'$clusterIssuerName'")') - -if [[ -n $result ]]; then - echo "[$clusterIssuerName] cluster issuer already exists" - exit -else - # Create the cluster issuer - echo "[$clusterIssuerName] cluster issuer does not exist" - echo "Creating [$clusterIssuerName] cluster issuer..." - cat $clusterIssuerTemplate | - yq "(.spec.acme.email)|="\""$email"\" | - kubectl apply -f - -fi diff --git a/scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh b/scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh index c770e6476..a8d5a6cf0 100644 --- a/scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh +++ b/scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh @@ -1,44 +1,14 @@ #!/bin/bash -# Variables -source ./00-variables.sh +openAiName="CoralOpenAi" +openAiResourceGroupName="CoralRG" +managedIdentityName="CyanWorkloadManagedIdentity" -# Check if the user-assigned managed identity already exists -echo "Checking if [$managedIdentityName] user-assigned managed identity actually exists in the [$aksResourceGroupName] resource group..." - -az identity show \ - --name $managedIdentityName \ - --resource-group $aksResourceGroupName &>/dev/null - -if [[ $? != 0 ]]; then - echo "No [$managedIdentityName] user-assigned managed identity actually exists in the [$aksResourceGroupName] resource group" - echo "Creating [$managedIdentityName] user-assigned managed identity in the [$aksResourceGroupName] resource group..." - - # Create the user-assigned managed identity - az identity create \ - --name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --location $location \ - --subscription $subscriptionId 1>/dev/null - - if [[ $? == 0 ]]; then - echo "[$managedIdentityName] user-assigned managed identity successfully created in the [$aksResourceGroupName] resource group" - else - echo "Failed to create [$managedIdentityName] user-assigned managed identity in the [$aksResourceGroupName] resource group" - exit - fi -else - echo "[$managedIdentityName] user-assigned managed identity already exists in the [$aksResourceGroupName] resource group" -fi - -# Retrieve the clientId of the user-assigned managed identity -echo "Retrieving clientId for [$managedIdentityName] managed identity..." clientId=$(az identity show \ --name $managedIdentityName \ --resource-group $aksResourceGroupName \ --query clientId \ --output tsv) - if [[ -n $clientId ]]; then echo "[$clientId] clientId for the [$managedIdentityName] managed identity successfully retrieved" else @@ -46,14 +16,11 @@ else exit fi -# Retrieve the principalId of the user-assigned managed identity -echo "Retrieving principalId for [$managedIdentityName] managed identity..." principalId=$(az identity show \ --name $managedIdentityName \ --resource-group $aksResourceGroupName \ --query principalId \ --output tsv) - if [[ -n $principalId ]]; then echo "[$principalId] principalId for the [$managedIdentityName] managed identity successfully retrieved" else @@ -61,13 +28,11 @@ else exit fi -# Get the resource id of the Azure OpenAI resource openAiId=$(az cognitiveservices account show \ --name $openAiName \ --resource-group $openAiResourceGroupName \ --query id \ --output tsv) - if [[ -n $openAiId ]]; then echo "Resource id for the [$openAiName] Azure OpenAI resource successfully retrieved" else diff --git a/scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh b/scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh index 5a89a0619..9d7f7aec4 100644 --- a/scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh +++ b/scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh @@ -1,103 +1,39 @@ #!/bin/bash +aksClusterName="CoralAks" -# Variables for the user-assigned managed identity -source ./00-variables.sh - -# Check if the namespace already exists -result=$(kubectl get namespace -o 'jsonpath={.items[?(@.metadata.name=="'$namespace'")].metadata.name'}) - -if [[ -n $result ]]; then - echo "[$namespace] namespace already exists" -else - # Create the namespace for your ingress resources - echo "[$namespace] namespace does not exist" - echo "Creating [$namespace] namespace..." - kubectl create namespace $namespace -fi - -# Check if the service account already exists -result=$(kubectl get sa -n $namespace -o 'jsonpath={.items[?(@.metadata.name=="'$serviceAccountName'")].metadata.name'}) +# Retrieve the resource id of the user-assigned managed identity +echo "Retrieving clientId for [$managedIdentityName] managed identity..." +managedIdentityClientId=$(az identity show \ + --name $managedIdentityName \ + --resource-group $aksResourceGroupName \ + --query clientId \ + --output tsv) -if [[ -n $result ]]; then - echo "[$serviceAccountName] service account already exists" +if [[ -n $managedIdentityClientId ]]; then + echo "[$managedIdentityClientId] clientId for the [$managedIdentityName] managed identity successfully retrieved" else - # Retrieve the resource id of the user-assigned managed identity - echo "Retrieving clientId for [$managedIdentityName] managed identity..." - managedIdentityClientId=$(az identity show \ - --name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --query clientId \ - --output tsv) - - if [[ -n $managedIdentityClientId ]]; then - echo "[$managedIdentityClientId] clientId for the [$managedIdentityName] managed identity successfully retrieved" - else - echo "Failed to retrieve clientId for the [$managedIdentityName] managed identity" - exit - fi - - # Create the service account - echo "[$serviceAccountName] service account does not exist" - echo "Creating [$serviceAccountName] service account..." - cat </dev/null - -if [[ $? != 0 ]]; then - echo "No [$federatedIdentityName] federated identity credential actually exists in the [$aksResourceGroupName] resource group" - - # Get the OIDC Issuer URL - aksOidcIssuerUrl="$(az aks show \ - --only-show-errors \ - --name $aksClusterName \ - --resource-group $aksResourceGroupName \ - --query oidcIssuerProfile.issuerUrl \ - --output tsv)" - - # Show OIDC Issuer URL - if [[ -n $aksOidcIssuerUrl ]]; then - echo "The OIDC Issuer URL of the $aksClusterName cluster is $aksOidcIssuerUrl" - fi - - echo "Creating [$federatedIdentityName] federated identity credential in the [$aksResourceGroupName] resource group..." - - # Establish the federated identity credential between the managed identity, the service account issuer, and the subject. - az identity federated-credential create \ - --name $federatedIdentityName \ - --identity-name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --issuer $aksOidcIssuerUrl \ - --subject system:serviceaccount:$namespace:$serviceAccountName - - if [[ $? == 0 ]]; then - echo "[$federatedIdentityName] federated identity credential successfully created in the [$aksResourceGroupName] resource group" - else - echo "Failed to create [$federatedIdentityName] federated identity credential in the [$aksResourceGroupName] resource group" - exit - fi + --issuer $aksOidcIssuerUrl \ + --subject system:serviceaccount:$namespace:$serviceAccountName +if [[ $? == 0 ]]; then + echo "[$federatedIdentityName] federated identity credential successfully created in the [$aksResourceGroupName] resource group" else - echo "[$federatedIdentityName] federated identity credential already exists in the [$aksResourceGroupName] resource group" -fi \ No newline at end of file + echo "Failed to create [$federatedIdentityName] federated identity credential in the [$aksResourceGroupName] resource group" + exit +fi diff --git a/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh b/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh index f9e1d757c..962b75302 100644 --- a/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh +++ b/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh @@ -1,18 +1,15 @@ #!/bin/bash -# Variables -source ./00-variables.sh +openAiBase="https://coralopenai.openai.azure.com/" +openAiType="azure_ad" +openAiModel="gpt-35-turbo" +openAiDeployment="gpt-35-turbo" -# Check if namespace exists in the cluster -result=$(kubectl get namespace -o jsonpath="{.items[?(@.metadata.name=='$namespace')].metadata.name}") - -if [[ -n $result ]]; then - echo "$namespace namespace already exists in the cluster" -else - echo "$namespace namespace does not exist in the cluster" - echo "creating $namespace namespace in the cluster..." - kubectl create namespace $namespace -fi +# Parameters +title="Magic 8 Ball" +label="Pose your question and cross your fingers!" +temperature="0.9" +imageWidth="80" # Create config map cat $configMapTemplate | diff --git a/scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh b/scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh deleted file mode 100644 index 52f090706..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/10-create-ingress.sh +++ /dev/null @@ -1,9 +0,0 @@ -#/bin/bash - -# Create the ingress -echo "[$ingressName] ingress does not exist" -echo "Creating [$ingressName] ingress..." -cat $ingressTemplate | - yq "(.spec.tls[0].hosts[0])|="\""$host"\" | - yq "(.spec.rules[0].host)|="\""$host"\" | - kubectl apply -n $namespace -f - \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh b/scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh index 95f8baf69..da0da17b0 100644 --- a/scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh +++ b/scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh @@ -1,5 +1,16 @@ -# Variables -source ./00-variables.sh +ingressTemplate="ingress.yml" +ingressName="magic8ball-ingress" +dnsZoneName="contoso.com" +dnsZoneResourceGroupName="DnsResourceGroup" +subdomain="magic8ball" +host="$subdomain.$dnsZoneName" + +echo "[$ingressName] ingress does not exist" +echo "Creating [$ingressName] ingress..." +cat $ingressTemplate | + yq "(.spec.tls[0].hosts[0])|="\""$host"\" | + yq "(.spec.rules[0].host)|="\""$host"\" | + kubectl apply -n $namespace -f - # Retrieve the public IP address from the ingress echo "Retrieving the external IP address from the [$ingressName] ingress..." diff --git a/scenarios/AksOpenAiTerraform/scripts/build-app-image.sh b/scenarios/AksOpenAiTerraform/scripts/build-app-image.sh new file mode 100644 index 000000000..c8de1b51c --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/build-app-image.sh @@ -0,0 +1,10 @@ +ACR_NAME=$(terraform output resource_group_name) +IMAGE_NAME="magic8ball" +TAG="v1" +IMAGE="$ACR_NAME.azurecr.io/$IMAGE_NAME:$TAG" + +az acr login --name $ACR_NAME + +# Build and push app image +ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) +docker build -t $ACR_URL/$IMAGE ./app --push \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/install-nginx-via-helm-and-create-sa.sh b/scenarios/AksOpenAiTerraform/scripts/install-nginx-via-helm-and-create-sa.sh index 45359f500..aa5ac2164 100644 --- a/scenarios/AksOpenAiTerraform/scripts/install-nginx-via-helm-and-create-sa.sh +++ b/scenarios/AksOpenAiTerraform/scripts/install-nginx-via-helm-and-create-sa.sh @@ -1,6 +1,3 @@ -# Install kubectl -az aks install-cli --only-show-errors - # Get AKS credentials az aks get-credentials \ --admin \ @@ -9,210 +6,40 @@ az aks get-credentials \ --subscription $subscriptionId \ --only-show-errors -# Check if the cluster is private or not -private=$(az aks show --name $clusterName \ - --resource-group $resourceGroupName \ - --subscription $subscriptionId \ - --query apiServerAccessProfile.enablePrivateCluster \ - --output tsv) - # Install Helm curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 -o get_helm.sh -s chmod 700 get_helm.sh ./get_helm.sh &>/dev/null -# Add Helm repos -helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +# NGINX ingress controller helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm install nginx-ingress ingress-nginx/ingress-nginx \ + --create-namespace \ + --namespace "ingress-basic" \ + --set controller.replicaCount=3 \ + --set controller.nodeSelector."kubernetes\.io/os"=linux \ + --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz \ + --set controller.metrics.enabled=true \ + --set controller.metrics.serviceMonitor.enabled=true \ + --set controller.metrics.serviceMonitor.additionalLabels.release="prometheus" \ + +# Cert manager helm repo add jetstack https://charts.jetstack.io - -# Update Helm repos -helm repo update - -if [[ $private == 'true' ]]; then - # Log whether the cluster is public or private - echo "$clusterName AKS cluster is public" - - # Install Prometheus - command="helm install prometheus prometheus-community/kube-prometheus-stack \ +helm install cert-manager jetstack/cert-manager \ --create-namespace \ - --namespace prometheus \ - --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ - --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false" - - az aks command invoke \ - --name $clusterName \ - --resource-group $resourceGroupName \ - --subscription $subscriptionId \ - --command "$command" - - # Install NGINX ingress controller using the internal load balancer - command="helm install nginx-ingress ingress-nginx/ingress-nginx \ - --create-namespace \ - --namespace ingress-basic \ - --set controller.replicaCount=3 \ - --set controller.nodeSelector.\"kubernetes\.io/os\"=linux \ - --set defaultBackend.nodeSelector.\"kubernetes\.io/os\"=linux \ - --set controller.metrics.enabled=true \ - --set controller.metrics.serviceMonitor.enabled=true \ - --set controller.metrics.serviceMonitor.additionalLabels.release=\"prometheus\" \ - --set controller.service.annotations.\"service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path\"=/healthz" - - az aks command invoke \ - --name $clusterName \ - --resource-group $resourceGroupName \ - --subscription $subscriptionId \ - --command "$command" - - # Install certificate manager - command="helm install cert-manager jetstack/cert-manager \ - --create-namespace \ - --namespace cert-manager \ - --set installCRDs=true \ - --set nodeSelector.\"kubernetes\.io/os\"=linux" - - az aks command invoke \ - --name $clusterName \ - --resource-group $resourceGroupName \ - --subscription $subscriptionId \ - --command "$command" - - # Create cluster issuer - command="cat <$AZ_SCRIPTS_OUTPUT_PATH \ No newline at end of file +kubectl create namespace $namespace # Create workload namespace +kubectl apply -f cluster-issuer.yml +kubectl apply -f serviceAccount.yml \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml index 6855fdf8c..ecfe13664 100644 --- a/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml +++ b/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml @@ -5,7 +5,7 @@ metadata: spec: acme: server: https://acme-v02.api.letsencrypt.org/directory - email: paolos@microsoft.com + email: $email privateKeySecretRef: name: letsencrypt solvers: diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/serviceAccount.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/serviceAccount.yml new file mode 100644 index 000000000..a5ab35826 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/manifests/serviceAccount.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + azure.workload.identity/client-id: $workloadManagedIdentityClientId + azure.workload.identity/tenant-id: $tenantId + labels: + azure.workload.identity/use: "true" + name: $serviceAccountName + namespace: $namespace \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh b/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh index af015f216..e1a5c792e 100644 --- a/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh +++ b/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh @@ -22,50 +22,4 @@ else echo "Failed to install [aks-preview] extension" exit fi -fi - -# Registering AKS features -aksExtensions=( - "AzureServiceMeshPreview" - "AKS-KedaPreview" - "RunCommandPreview" - "EnableOIDCIssuerPreview" - "EnableWorkloadIdentityPreview" - "EnableImageCleanerPreview" -"AKS-VPAPreview") -ok=0 -registeringExtensions=() -for aksExtension in ${aksExtensions[@]}; do - echo "Checking if [$aksExtension] extension is already registered..." - extension=$(az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService/$aksExtension') && @.properties.state == 'Registered'].{Name:name}" --output tsv) - if [[ -z $extension ]]; then - echo "[$aksExtension] extension is not registered." - echo "Registering [$aksExtension] extension..." - az feature register --name $aksExtension --namespace Microsoft.ContainerService - registeringExtensions+=("$aksExtension") - ok=1 - else - echo "[$aksExtension] extension is already registered." - fi -done -echo $registeringExtensions -delay=1 -for aksExtension in ${registeringExtensions[@]}; do - echo -n "Checking if [$aksExtension] extension is already registered..." - while true; do - extension=$(az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService/$aksExtension') && @.properties.state == 'Registered'].{Name:name}" --output tsv) - if [[ -z $extension ]]; then - echo -n "." - sleep $delay - else - echo "." - break - fi - done -done - -if [[ $ok == 1 ]]; then - echo "Refreshing the registration of the Microsoft.ContainerService resource provider..." - az provider register --namespace Microsoft.ContainerService - echo "Microsoft.ContainerService resource provider registration successfully refreshed" fi \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf index c9c982639..01f2301cc 100644 --- a/scenarios/AksOpenAiTerraform/terraform/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/outputs.tf @@ -1,3 +1,7 @@ +output "resource_group_name" { + value = module.azurerm_resource_group.name +} + output "acr_url" { value = module.container_registry.name } \ No newline at end of file From 3f29ae66c5dba616b6744abf838c71e0b5f4b669 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 24 Feb 2025 22:39:36 -0500 Subject: [PATCH 076/119] WIP --- .../07-create-workload-managed-identity.sh | 69 -------------- .../scripts/08-create-service-account.sh | 39 -------- .../scripts/11-configure-dns.sh | 90 ------------------- .../{build-app-image.sh => build-image.sh} | 0 .../{09-deploy-app.sh => deploy-app.sh} | 4 +- scenarios/AksOpenAiTerraform/scripts/dns.sh | 14 +++ ...x-via-helm-and-create-sa.sh => install.sh} | 3 +- .../scripts/register-preview-features.sh | 4 +- .../scripts/{00-variables.sh => vars.sh} | 7 +- 9 files changed, 21 insertions(+), 209 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh rename scenarios/AksOpenAiTerraform/scripts/{build-app-image.sh => build-image.sh} (100%) rename scenarios/AksOpenAiTerraform/scripts/{09-deploy-app.sh => deploy-app.sh} (95%) create mode 100644 scenarios/AksOpenAiTerraform/scripts/dns.sh rename scenarios/AksOpenAiTerraform/scripts/{install-nginx-via-helm-and-create-sa.sh => install.sh} (95%) rename scenarios/AksOpenAiTerraform/scripts/{00-variables.sh => vars.sh} (65%) diff --git a/scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh b/scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh deleted file mode 100644 index a8d5a6cf0..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/07-create-workload-managed-identity.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -openAiName="CoralOpenAi" -openAiResourceGroupName="CoralRG" -managedIdentityName="CyanWorkloadManagedIdentity" - -clientId=$(az identity show \ - --name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --query clientId \ - --output tsv) -if [[ -n $clientId ]]; then - echo "[$clientId] clientId for the [$managedIdentityName] managed identity successfully retrieved" -else - echo "Failed to retrieve clientId for the [$managedIdentityName] managed identity" - exit -fi - -principalId=$(az identity show \ - --name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --query principalId \ - --output tsv) -if [[ -n $principalId ]]; then - echo "[$principalId] principalId for the [$managedIdentityName] managed identity successfully retrieved" -else - echo "Failed to retrieve principalId for the [$managedIdentityName] managed identity" - exit -fi - -openAiId=$(az cognitiveservices account show \ - --name $openAiName \ - --resource-group $openAiResourceGroupName \ - --query id \ - --output tsv) -if [[ -n $openAiId ]]; then - echo "Resource id for the [$openAiName] Azure OpenAI resource successfully retrieved" -else - echo "Failed to the resource id for the [$openAiName] Azure OpenAI resource" - exit -1 -fi - -# Assign the Cognitive Services User role on the Azure OpenAI resource to the managed identity -role="Cognitive Services User" -echo "Checking if the [$managedIdentityName] managed identity has been assigned to [$role] role with [$openAiName] Azure OpenAI resource as a scope..." -current=$(az role assignment list \ - --assignee $principalId \ - --scope $openAiId \ - --query "[?roleDefinitionName=='$role'].roleDefinitionName" \ - --output tsv 2>/dev/null) - -if [[ $current == $role ]]; then - echo "[$managedIdentityName] managed identity is already assigned to the ["$current"] role with [$openAiName] Azure OpenAI resource as a scope" -else - echo "[$managedIdentityName] managed identity is not assigned to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" - echo "Assigning the [$role] role to the [$managedIdentityName] managed identity with [$openAiName] Azure OpenAI resource as a scope..." - - az role assignment create \ - --assignee $principalId \ - --role "$role" \ - --scope $openAiId 1>/dev/null - - if [[ $? == 0 ]]; then - echo "[$managedIdentityName] managed identity successfully assigned to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" - else - echo "Failed to assign the [$managedIdentityName] managed identity to the [$role] role with [$openAiName] Azure OpenAI resource as a scope" - exit - fi -fi \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh b/scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh deleted file mode 100644 index 9d7f7aec4..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/08-create-service-account.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -aksClusterName="CoralAks" - -# Retrieve the resource id of the user-assigned managed identity -echo "Retrieving clientId for [$managedIdentityName] managed identity..." -managedIdentityClientId=$(az identity show \ - --name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --query clientId \ - --output tsv) - -if [[ -n $managedIdentityClientId ]]; then - echo "[$managedIdentityClientId] clientId for the [$managedIdentityName] managed identity successfully retrieved" -else - echo "Failed to retrieve clientId for the [$managedIdentityName] managed identity" - exit -fi - -# Get the OIDC Issuer URL -aksOidcIssuerUrl="$(az aks show \ - --only-show-errors \ - --name $aksClusterName \ - --resource-group $aksResourceGroupName \ - --query oidcIssuerProfile.issuerUrl \ - --output tsv)" - -# Establish the federated identity credential between the managed identity, the service account issuer, and the subject. -az identity federated-credential create \ - --name $federatedIdentityName \ - --identity-name $managedIdentityName \ - --resource-group $aksResourceGroupName \ - --issuer $aksOidcIssuerUrl \ - --subject system:serviceaccount:$namespace:$serviceAccountName -if [[ $? == 0 ]]; then - echo "[$federatedIdentityName] federated identity credential successfully created in the [$aksResourceGroupName] resource group" -else - echo "Failed to create [$federatedIdentityName] federated identity credential in the [$aksResourceGroupName] resource group" - exit -fi diff --git a/scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh b/scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh deleted file mode 100644 index da0da17b0..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/11-configure-dns.sh +++ /dev/null @@ -1,90 +0,0 @@ -ingressTemplate="ingress.yml" -ingressName="magic8ball-ingress" -dnsZoneName="contoso.com" -dnsZoneResourceGroupName="DnsResourceGroup" -subdomain="magic8ball" -host="$subdomain.$dnsZoneName" - -echo "[$ingressName] ingress does not exist" -echo "Creating [$ingressName] ingress..." -cat $ingressTemplate | - yq "(.spec.tls[0].hosts[0])|="\""$host"\" | - yq "(.spec.rules[0].host)|="\""$host"\" | - kubectl apply -n $namespace -f - - -# Retrieve the public IP address from the ingress -echo "Retrieving the external IP address from the [$ingressName] ingress..." -publicIpAddress=$(kubectl get ingress $ingressName -n $namespace -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - -if [ -n $publicIpAddress ]; then - echo "[$publicIpAddress] external IP address of the application gateway ingress controller successfully retrieved from the [$ingressName] ingress" -else - echo "Failed to retrieve the external IP address of the application gateway ingress controller from the [$ingressName] ingress" - exit -fi - -# Check if an A record for todolist subdomain exists in the DNS Zone -echo "Retrieving the A record for the [$subdomain] subdomain from the [$dnsZoneName] DNS zone..." -ipv4Address=$(az network dns record-set a list \ - --zone-name $dnsZoneName \ - --resource-group $dnsZoneResourceGroupName \ - --query "[?name=='$subdomain'].arecords[].ipv4Address" \ - --output tsv) - -if [[ -n $ipv4Address ]]; then - echo "An A record already exists in [$dnsZoneName] DNS zone for the [$subdomain] subdomain with [$ipv4Address] IP address" - - if [[ $ipv4Address == $publicIpAddress ]]; then - echo "The [$ipv4Address] ip address of the existing A record is equal to the ip address of the [$ingressName] ingress" - echo "No additional step is required" - exit - else - echo "The [$ipv4Address] ip address of the existing A record is different than the ip address of the [$ingressName] ingress" - fi - - # Retrieving name of the record set relative to the zone - echo "Retrieving the name of the record set relative to the [$dnsZoneName] zone..." - - recordSetName=$(az network dns record-set a list \ - --zone-name $dnsZoneName \ - --resource-group $dnsZoneResourceGroupName \ - --query "[?name=='$subdomain'].name" \ - --output name 2>/dev/null) - - if [[ -n $recordSetName ]]; then - "[$recordSetName] record set name successfully retrieved" - else - "Failed to retrieve the name of the record set relative to the [$dnsZoneName] zone" - exit - fi - - # Remove the a record - echo "Removing the A record from the record set relative to the [$dnsZoneName] zone..." - - az network dns record-set a remove-record \ - --ipv4-address $ipv4Address \ - --record-set-name $recordSetName \ - --zone-name $dnsZoneName \ - --resource-group $dnsZoneResourceGroupName - - if [[ $? == 0 ]]; then - echo "[$ipv4Address] ip address successfully removed from the [$recordSetName] record set" - else - echo "Failed to remove the [$ipv4Address] ip address from the [$recordSetName] record set" - exit - fi -fi - -# Create the a record -echo "Creating an A record in [$dnsZoneName] DNS zone for the [$subdomain] subdomain with [$publicIpAddress] IP address..." -az network dns record-set a add-record \ - --zone-name $dnsZoneName \ - --resource-group $dnsZoneResourceGroupName \ - --record-set-name $subdomain \ - --ipv4-address $publicIpAddress 1>/dev/null - -if [[ $? == 0 ]]; then - echo "A record for the [$subdomain] subdomain with [$publicIpAddress] IP address successfully created in [$dnsZoneName] DNS zone" -else - echo "Failed to create an A record for the $subdomain subdomain with [$publicIpAddress] IP address in [$dnsZoneName] DNS zone" -fi diff --git a/scenarios/AksOpenAiTerraform/scripts/build-app-image.sh b/scenarios/AksOpenAiTerraform/scripts/build-image.sh similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/build-app-image.sh rename to scenarios/AksOpenAiTerraform/scripts/build-image.sh diff --git a/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh b/scenarios/AksOpenAiTerraform/scripts/deploy-app.sh similarity index 95% rename from scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh rename to scenarios/AksOpenAiTerraform/scripts/deploy-app.sh index 962b75302..1b85713e6 100644 --- a/scenarios/AksOpenAiTerraform/scripts/09-deploy-app.sh +++ b/scenarios/AksOpenAiTerraform/scripts/deploy-app.sh @@ -12,7 +12,7 @@ temperature="0.9" imageWidth="80" # Create config map -cat $configMapTemplate | +cat configMap.yml | yq "(.data.TITLE)|="\""$title"\" | yq "(.data.LABEL)|="\""$label"\" | yq "(.data.TEMPERATURE)|="\""$temperature"\" | @@ -24,7 +24,7 @@ cat $configMapTemplate | kubectl apply -n $namespace -f - # Create deployment -cat $deploymentTemplate | +cat deployment.yml | yq "(.spec.template.spec.containers[0].image)|="\""$image"\" | yq "(.spec.template.spec.containers[0].imagePullPolicy)|="\""$imagePullPolicy"\" | yq "(.spec.template.spec.serviceAccountName)|="\""$serviceAccountName"\" | diff --git a/scenarios/AksOpenAiTerraform/scripts/dns.sh b/scenarios/AksOpenAiTerraform/scripts/dns.sh new file mode 100644 index 000000000..268c7a1ec --- /dev/null +++ b/scenarios/AksOpenAiTerraform/scripts/dns.sh @@ -0,0 +1,14 @@ +ingressName="magic8ball-ingress" +publicIpAddress=$(kubectl get ingress $ingressName -n $namespace -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +if [ -n $publicIpAddress ]; then + echo "[$publicIpAddress] external IP address of the application gateway ingress controller successfully retrieved from the [$ingressName] ingress" +else + echo "Failed to retrieve the external IP address of the application gateway ingress controller from the [$ingressName] ingress" + exit +fi + +az network dns record-set a add-record \ + --zone-name "contoso.com" \ + --resource-group $RESOURCE_GROUP \ + --record-set-name magic8ball \ + --ipv4-address $publicIpAddress \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/install-nginx-via-helm-and-create-sa.sh b/scenarios/AksOpenAiTerraform/scripts/install.sh similarity index 95% rename from scenarios/AksOpenAiTerraform/scripts/install-nginx-via-helm-and-create-sa.sh rename to scenarios/AksOpenAiTerraform/scripts/install.sh index aa5ac2164..dcf0e702f 100644 --- a/scenarios/AksOpenAiTerraform/scripts/install-nginx-via-helm-and-create-sa.sh +++ b/scenarios/AksOpenAiTerraform/scripts/install.sh @@ -42,4 +42,5 @@ helm install prometheus prometheus-community/kube-prometheus-stack \ kubectl create namespace $namespace # Create workload namespace kubectl apply -f cluster-issuer.yml -kubectl apply -f serviceAccount.yml \ No newline at end of file +kubectl apply -f serviceAccount.yml +kubectl apply -n $namespace -f ingress.yml \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh b/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh index e1a5c792e..2abdce2a7 100644 --- a/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh +++ b/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh @@ -2,14 +2,14 @@ # Install aks-preview Azure extension echo "Checking if [aks-preview] extension is already installed..." -az extension show --name aks-preview &>/dev/null +az extension show --name aks-preview if [[ $? == 0 ]]; then echo "[aks-preview] extension is already installed" # Update the extension to make sure you have the latest version installed echo "Updating [aks-preview] extension..." - az extension update --name aks-preview &>/dev/null + az extension update --name aks-preview else echo "[aks-preview] extension is not installed. Installing..." diff --git a/scenarios/AksOpenAiTerraform/scripts/00-variables.sh b/scenarios/AksOpenAiTerraform/scripts/vars.sh similarity index 65% rename from scenarios/AksOpenAiTerraform/scripts/00-variables.sh rename to scenarios/AksOpenAiTerraform/scripts/vars.sh index dba29b422..fd5a741c1 100644 --- a/scenarios/AksOpenAiTerraform/scripts/00-variables.sh +++ b/scenarios/AksOpenAiTerraform/scripts/vars.sh @@ -10,9 +10,4 @@ aksResourceGroupName="CoralRG" # Sample Application namespace="magic8ball" -serviceAccountName="magic8ball-sa" - -deploymentTemplate="deployment.yml" -serviceTemplate="service.yml" -configMapTemplate="configMap.yml" -secretTemplate="secret.yml" \ No newline at end of file +serviceAccountName="magic8ball-sa" \ No newline at end of file From fd3b1785d5f53a15b7491f5fd4fd9c946db99e9e Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 24 Feb 2025 22:46:48 -0500 Subject: [PATCH 077/119] Rename --- .../AksOpenAiTerraform/terraform/main.tf | 36 +++++++++---------- .../AksOpenAiTerraform/terraform/outputs.tf | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 4a9dd027d..836c4dfb2 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -29,7 +29,7 @@ locals { ############################################################################### # Resource Group ############################################################################### -resource "azurerm_resource_group" "rg" { +resource "azurerm_resource_group" "main" { name = "${var.resource_group_name_prefix}-${local.random_id}-rg" location = var.location @@ -45,7 +45,7 @@ module "openai" { source = "./modules/openai" name = "OpenAi-${local.random_id}" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name sku_name = "S0" deployments = [ @@ -66,8 +66,8 @@ module "aks_cluster" { source = "./modules/aks" name = "AksCluster" location = var.location - resource_group_name = azurerm_resource_group.rg.name - resource_group_id = azurerm_resource_group.rg.id + resource_group_name = azurerm_resource_group.main.name + resource_group_id = azurerm_resource_group.main.id tenant_id = local.tenant_id kubernetes_version = var.kubernetes_version @@ -88,7 +88,7 @@ module "container_registry" { source = "./modules/container_registry" name = "acr${local.random_id}" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name sku = "Premium" @@ -99,14 +99,14 @@ module "storage_account" { source = "./modules/storage_account" name = "boot${random_string.storage_account_suffix.result}" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name } module "key_vault" { source = "./modules/key_vault" name = "KeyVault-${local.random_id}" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name tenant_id = local.tenant_id sku_name = "standard" @@ -118,7 +118,7 @@ module "log_analytics_workspace" { source = "./modules/log_analytics" name = "Workspace" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name sku = "PerGB2018" retention_in_days = 30 @@ -131,7 +131,7 @@ module "virtual_network" { source = "./modules/virtual_network" name = "AksVNet" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name address_space = ["10.0.0.0/8"] subnets = [ @@ -171,7 +171,7 @@ module "nat_gateway" { source = "./modules/nat_gateway" name = "NatGateway" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name subnet_ids = module.virtual_network.subnet_ids } @@ -180,7 +180,7 @@ module "bastion_host" { source = "./modules/bastion_host" name = "BastionHost" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name subnet_id = module.virtual_network.subnet_ids["AzureBastionSubnet"] @@ -193,7 +193,7 @@ module "bastion_host" { module "acr_private_dns_zone" { source = "./modules/dns" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name name = "privatelink.azurecr.io" subresource_name = "account" @@ -205,7 +205,7 @@ module "acr_private_dns_zone" { module "openai_private_dns_zone" { source = "./modules/dns" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name name = "privatelink.openai.azure.com" subresource_name = "registry" @@ -217,7 +217,7 @@ module "openai_private_dns_zone" { module "key_vault_private_dns_zone" { source = "./modules/dns" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name name = "privatelink.vaultcore.azure.net" subresource_name = "vault" @@ -229,7 +229,7 @@ module "key_vault_private_dns_zone" { module "blob_private_dns_zone" { source = "./modules/dns" location = var.location - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name name = "privatelink.blob.core.windows.net" subresource_name = "blob" @@ -243,13 +243,13 @@ module "blob_private_dns_zone" { ############################################################################### resource "azurerm_user_assigned_identity" "aks_workload_identity" { name = "WorkloadManagedIdentity" - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name location = var.location } resource "azurerm_federated_identity_credential" "federated_identity_credential" { name = "${title(local.namespace)}FederatedIdentity" - resource_group_name = azurerm_resource_group.rg.name + resource_group_name = azurerm_resource_group.main.name audience = ["api://AzureADTokenExchange"] issuer = module.aks_cluster.oidc_issuer_url @@ -265,7 +265,7 @@ resource "azurerm_role_assignment" "cognitive_services_user_assignment" { resource "azurerm_role_assignment" "network_contributor_assignment" { role_definition_name = "Network Contributor" - scope = azurerm_resource_group.rg.id + scope = azurerm_resource_group.main.id principal_id = module.aks_cluster.aks_identity_principal_id } diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf index 01f2301cc..9f0f3f4bf 100644 --- a/scenarios/AksOpenAiTerraform/terraform/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/outputs.tf @@ -1,5 +1,5 @@ output "resource_group_name" { - value = module.azurerm_resource_group.name + value = azurerm_resource_group.main.name } output "acr_url" { From 805a02f96c6ec14cf6cb0dba5edf79af5daa1699 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 25 Feb 2025 00:18:03 -0500 Subject: [PATCH 078/119] Wip --- .../AksOpenAiTerraform/scripts/deploy-app.sh | 2 +- .../AksOpenAiTerraform/scripts/install.sh | 2 +- .../scripts/manifests/cluster-issuer.yml | 2 +- .../{configMap.yml => config-map.yml} | 0 ...serviceAccount.yml => service-account.yml} | 0 scenarios/AksOpenAiTerraform/scripts/vars.sh | 7 ++--- .../AksOpenAiTerraform/terraform/main.tf | 8 +++--- .../terraform/modules/aks/main.tf | 28 ++----------------- .../terraform/modules/aks/outputs.tf | 2 +- 9 files changed, 13 insertions(+), 38 deletions(-) rename scenarios/AksOpenAiTerraform/scripts/manifests/{configMap.yml => config-map.yml} (100%) rename scenarios/AksOpenAiTerraform/scripts/manifests/{serviceAccount.yml => service-account.yml} (100%) diff --git a/scenarios/AksOpenAiTerraform/scripts/deploy-app.sh b/scenarios/AksOpenAiTerraform/scripts/deploy-app.sh index 1b85713e6..acb922e19 100644 --- a/scenarios/AksOpenAiTerraform/scripts/deploy-app.sh +++ b/scenarios/AksOpenAiTerraform/scripts/deploy-app.sh @@ -12,7 +12,7 @@ temperature="0.9" imageWidth="80" # Create config map -cat configMap.yml | +cat config-map.yml | yq "(.data.TITLE)|="\""$title"\" | yq "(.data.LABEL)|="\""$label"\" | yq "(.data.TEMPERATURE)|="\""$temperature"\" | diff --git a/scenarios/AksOpenAiTerraform/scripts/install.sh b/scenarios/AksOpenAiTerraform/scripts/install.sh index dcf0e702f..0007728d1 100644 --- a/scenarios/AksOpenAiTerraform/scripts/install.sh +++ b/scenarios/AksOpenAiTerraform/scripts/install.sh @@ -42,5 +42,5 @@ helm install prometheus prometheus-community/kube-prometheus-stack \ kubectl create namespace $namespace # Create workload namespace kubectl apply -f cluster-issuer.yml -kubectl apply -f serviceAccount.yml +kubectl apply -f service-account.yml kubectl apply -n $namespace -f ingress.yml \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml index ecfe13664..6cc55451f 100644 --- a/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml +++ b/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml @@ -5,7 +5,7 @@ metadata: spec: acme: server: https://acme-v02.api.letsencrypt.org/directory - email: $email + email: {{ .Values.email }} privateKeySecretRef: name: letsencrypt solvers: diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/configMap.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/config-map.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/manifests/configMap.yml rename to scenarios/AksOpenAiTerraform/scripts/manifests/config-map.yml diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/serviceAccount.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/service-account.yml similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/manifests/serviceAccount.yml rename to scenarios/AksOpenAiTerraform/scripts/manifests/service-account.yml diff --git a/scenarios/AksOpenAiTerraform/scripts/vars.sh b/scenarios/AksOpenAiTerraform/scripts/vars.sh index fd5a741c1..4af9bf3b3 100644 --- a/scenarios/AksOpenAiTerraform/scripts/vars.sh +++ b/scenarios/AksOpenAiTerraform/scripts/vars.sh @@ -1,13 +1,10 @@ +cd terraform + RESOURCE_GROUP=$($(terraform output resource_group_name)) LOCATION="westus3" SUBSCRIPTION_ID=$(az account show --query id --output tsv) TENANT_ID=$(az account show --query tenantId --output tsv) -email="paolos@microsoft.com" - -# AKS Cluster -aksResourceGroupName="CoralRG" - # Sample Application namespace="magic8ball" serviceAccountName="magic8ball-sa" \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 836c4dfb2..80d57ec04 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -241,26 +241,26 @@ module "blob_private_dns_zone" { ############################################################################### # Identities/Roles ############################################################################### -resource "azurerm_user_assigned_identity" "aks_workload_identity" { +resource "azurerm_user_assigned_identity" "aks_workload" { name = "WorkloadManagedIdentity" resource_group_name = azurerm_resource_group.main.name location = var.location } -resource "azurerm_federated_identity_credential" "federated_identity_credential" { +resource "azurerm_federated_identity_credential" "this" { name = "${title(local.namespace)}FederatedIdentity" resource_group_name = azurerm_resource_group.main.name audience = ["api://AzureADTokenExchange"] issuer = module.aks_cluster.oidc_issuer_url - parent_id = azurerm_user_assigned_identity.aks_workload_identity.id + parent_id = azurerm_user_assigned_identity.aks_workload.id subject = "system:serviceaccount:${local.namespace}:${local.service_account_name}" } resource "azurerm_role_assignment" "cognitive_services_user_assignment" { role_definition_name = "Cognitive Services User" scope = module.openai.id - principal_id = azurerm_user_assigned_identity.aks_workload_identity.principal_id + principal_id = azurerm_user_assigned_identity.aks_workload.principal_id } resource "azurerm_role_assignment" "network_contributor_assignment" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index fc527fb9e..7d0946ad0 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -2,7 +2,7 @@ locals { zones = ["2", "3"] } -resource "azurerm_user_assigned_identity" "aks_identity" { +resource "azurerm_user_assigned_identity" "aks" { name = "${var.name}Identity" resource_group_name = var.resource_group_name location = var.location @@ -14,16 +14,11 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { resource_group_name = var.resource_group_name kubernetes_version = var.kubernetes_version dns_prefix = lower(var.name) - private_cluster_enabled = false automatic_upgrade_channel = "stable" sku_tier = var.sku_tier - workload_identity_enabled = true - oidc_issuer_enabled = true - open_service_mesh_enabled = true + image_cleaner_enabled = true image_cleaner_interval_hours = 72 - azure_policy_enabled = true - http_application_routing_enabled = false default_node_pool { name = "system" @@ -32,18 +27,11 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { vnet_subnet_id = var.system_node_pool_subnet_id pod_subnet_id = var.pod_subnet_id zones = local.zones - max_pods = 50 - - upgrade_settings { - drain_timeout_in_minutes = 0 - max_surge = "10%" - node_soak_duration_in_minutes = 0 - } } identity { type = "UserAssigned" - identity_ids = tolist([azurerm_user_assigned_identity.aks_identity.id]) + identity_ids = tolist([azurerm_user_assigned_identity.aks.id]) } network_profile { @@ -62,15 +50,6 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { tenant_id = var.tenant_id azure_rbac_enabled = true } - - workload_autoscaler_profile { - keda_enabled = true - vertical_pod_autoscaler_enabled = true - } - - lifecycle { - ignore_changes = [microsoft_defender] - } } resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { @@ -82,7 +61,6 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { vnet_subnet_id = var.user_node_pool_subnet_id pod_subnet_id = var.pod_subnet_id orchestrator_version = var.kubernetes_version - max_pods = 50 os_type = "Linux" priority = "Regular" } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf index 56139a135..80519e8d5 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf @@ -7,7 +7,7 @@ output "id" { } output "aks_identity_principal_id" { - value = azurerm_user_assigned_identity.aks_identity.principal_id + value = azurerm_user_assigned_identity.aks.id } output "kubelet_identity_object_id" { From 2eadfac849c3d46d9351103c36001f048b55a3d2 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 25 Feb 2025 00:42:28 -0500 Subject: [PATCH 079/119] WIP --- .../AksOpenAiTerraform/scripts/build-image.sh | 10 ---- .../AksOpenAiTerraform/scripts/deploy-app.sh | 34 ------------- .../scripts/{install.sh => deploy.sh} | 41 ++++++++++++++-- scenarios/AksOpenAiTerraform/scripts/dns.sh | 14 ------ .../scripts/manifests/deployment.yml | 49 ++----------------- scenarios/AksOpenAiTerraform/scripts/vars.sh | 10 ---- .../terraform/modules/aks/main.tf | 3 ++ .../terraform/modules/aks/outputs.tf | 2 +- 8 files changed, 43 insertions(+), 120 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/scripts/build-image.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/deploy-app.sh rename scenarios/AksOpenAiTerraform/scripts/{install.sh => deploy.sh} (53%) delete mode 100644 scenarios/AksOpenAiTerraform/scripts/dns.sh delete mode 100644 scenarios/AksOpenAiTerraform/scripts/vars.sh diff --git a/scenarios/AksOpenAiTerraform/scripts/build-image.sh b/scenarios/AksOpenAiTerraform/scripts/build-image.sh deleted file mode 100644 index c8de1b51c..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/build-image.sh +++ /dev/null @@ -1,10 +0,0 @@ -ACR_NAME=$(terraform output resource_group_name) -IMAGE_NAME="magic8ball" -TAG="v1" -IMAGE="$ACR_NAME.azurecr.io/$IMAGE_NAME:$TAG" - -az acr login --name $ACR_NAME - -# Build and push app image -ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) -docker build -t $ACR_URL/$IMAGE ./app --push \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/deploy-app.sh b/scenarios/AksOpenAiTerraform/scripts/deploy-app.sh deleted file mode 100644 index acb922e19..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/deploy-app.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -openAiBase="https://coralopenai.openai.azure.com/" -openAiType="azure_ad" -openAiModel="gpt-35-turbo" -openAiDeployment="gpt-35-turbo" - -# Parameters -title="Magic 8 Ball" -label="Pose your question and cross your fingers!" -temperature="0.9" -imageWidth="80" - -# Create config map -cat config-map.yml | - yq "(.data.TITLE)|="\""$title"\" | - yq "(.data.LABEL)|="\""$label"\" | - yq "(.data.TEMPERATURE)|="\""$temperature"\" | - yq "(.data.IMAGE_WIDTH)|="\""$imageWidth"\" | - yq "(.data.AZURE_OPENAI_TYPE)|="\""$openAiType"\" | - yq "(.data.AZURE_OPENAI_BASE)|="\""$openAiBase"\" | - yq "(.data.AZURE_OPENAI_MODEL)|="\""$openAiModel"\" | - yq "(.data.AZURE_OPENAI_DEPLOYMENT)|="\""$openAiDeployment"\" | - kubectl apply -n $namespace -f - - -# Create deployment -cat deployment.yml | - yq "(.spec.template.spec.containers[0].image)|="\""$image"\" | - yq "(.spec.template.spec.containers[0].imagePullPolicy)|="\""$imagePullPolicy"\" | - yq "(.spec.template.spec.serviceAccountName)|="\""$serviceAccountName"\" | - kubectl apply -n $namespace -f - - -# Create deployment -kubectl apply -f $serviceTemplate -n $namespace \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/install.sh b/scenarios/AksOpenAiTerraform/scripts/deploy.sh similarity index 53% rename from scenarios/AksOpenAiTerraform/scripts/install.sh rename to scenarios/AksOpenAiTerraform/scripts/deploy.sh index 0007728d1..b60748fa8 100644 --- a/scenarios/AksOpenAiTerraform/scripts/install.sh +++ b/scenarios/AksOpenAiTerraform/scripts/deploy.sh @@ -1,3 +1,15 @@ +# Variables +SUBSCRIPTION_ID=$(az account show --query id --output tsv) +TENANT_ID=$(az account show --query tenantId --output tsv) +RESOURCE_GROUP=$(terraform output resource_group_name) +LOCATION="westus3" + +# Build/Push App's Docker image +ACR_NAME=$(terraform output resource_group_name) +az acr login --name $ACR_NAME +ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) +docker build -t $ACR_URL/$ACR_NAME.azurecr.io/magic8ball:v1 ./app --push + # Get AKS credentials az aks get-credentials \ --admin \ @@ -11,7 +23,7 @@ curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 -o get_ chmod 700 get_helm.sh ./get_helm.sh &>/dev/null -# NGINX ingress controller +# Install NGINX ingress controller helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx helm install nginx-ingress ingress-nginx/ingress-nginx \ --create-namespace \ @@ -24,7 +36,7 @@ helm install nginx-ingress ingress-nginx/ingress-nginx \ --set controller.metrics.serviceMonitor.enabled=true \ --set controller.metrics.serviceMonitor.additionalLabels.release="prometheus" \ -# Cert manager +# Install Cert manager helm repo add jetstack https://charts.jetstack.io helm install cert-manager jetstack/cert-manager \ --create-namespace \ @@ -32,7 +44,7 @@ helm install cert-manager jetstack/cert-manager \ --set installCRDs=true \ --set nodeSelector."kubernetes\.io/os"=linux -# Prometheus +# Install Prometheus helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm install prometheus prometheus-community/kube-prometheus-stack \ --create-namespace \ @@ -40,7 +52,26 @@ helm install prometheus prometheus-community/kube-prometheus-stack \ --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false -kubectl create namespace $namespace # Create workload namespace +NAMESPACE="magic8ball" +kubectl create namespace $NAMESPACE kubectl apply -f cluster-issuer.yml kubectl apply -f service-account.yml -kubectl apply -n $namespace -f ingress.yml \ No newline at end of file +kubectl apply -n $NAMESPACE -f ingress.yml +kubectl apply -n $NAMESPACE -f config-map.yml +kubectl apply -n $NAMESPACE -f deployment.yml +kubectl apply -f "service.yml" -n $NAMESPACE + +# Add DNS Record +ingressName="magic8ball-ingress" +publicIpAddress=$(kubectl get ingress $ingressName -n $namespace -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +if [ -n $publicIpAddress ]; then + echo "[$publicIpAddress] external IP address of the application gateway ingress controller successfully retrieved from the [$ingressName] ingress" +else + echo "Failed to retrieve the external IP address of the application gateway ingress controller from the [$ingressName] ingress" + exit +fi +az network dns record-set a add-record \ + --zone-name "contoso.com" \ + --resource-group $RESOURCE_GROUP \ + --record-set-name magic8ball \ + --ipv4-address $publicIpAddress \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/dns.sh b/scenarios/AksOpenAiTerraform/scripts/dns.sh deleted file mode 100644 index 268c7a1ec..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/dns.sh +++ /dev/null @@ -1,14 +0,0 @@ -ingressName="magic8ball-ingress" -publicIpAddress=$(kubectl get ingress $ingressName -n $namespace -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -if [ -n $publicIpAddress ]; then - echo "[$publicIpAddress] external IP address of the application gateway ingress controller successfully retrieved from the [$ingressName] ingress" -else - echo "Failed to retrieve the external IP address of the application gateway ingress controller from the [$ingressName] ingress" - exit -fi - -az network dns record-set a add-record \ - --zone-name "contoso.com" \ - --resource-group $RESOURCE_GROUP \ - --record-set-name magic8ball \ - --ipv4-address $publicIpAddress \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/deployment.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/deployment.yml index afffab8df..ee805c4aa 100644 --- a/scenarios/AksOpenAiTerraform/scripts/manifests/deployment.yml +++ b/scenarios/AksOpenAiTerraform/scripts/manifests/deployment.yml @@ -75,49 +75,6 @@ spec: initialDelaySeconds: 60 periodSeconds: 30 timeoutSeconds: 5 - env: - - name: TITLE - valueFrom: - configMapKeyRef: - name: magic8ball-configmap - key: TITLE - - name: IMAGE_WIDTH - valueFrom: - configMapKeyRef: - name: magic8ball-configmap - key: IMAGE_WIDTH - - name: LABEL - valueFrom: - configMapKeyRef: - name: magic8ball-configmap - key: LABEL - - name: TEMPERATURE - valueFrom: - configMapKeyRef: - name: magic8ball-configmap - key: TEMPERATURE - - name: AZURE_OPENAI_TYPE - valueFrom: - configMapKeyRef: - name: magic8ball-configmap - key: AZURE_OPENAI_TYPE - - name: AZURE_OPENAI_BASE - valueFrom: - configMapKeyRef: - name: magic8ball-configmap - key: AZURE_OPENAI_BASE - - name: AZURE_OPENAI_KEY - valueFrom: - configMapKeyRef: - name: magic8ball-configmap - key: AZURE_OPENAI_KEY - - name: AZURE_OPENAI_MODEL - valueFrom: - configMapKeyRef: - name: magic8ball-configmap - key: AZURE_OPENAI_MODEL - - name: AZURE_OPENAI_DEPLOYMENT - valueFrom: - configMapKeyRef: - name: magic8ball-configmap - key: AZURE_OPENAI_DEPLOYMENT \ No newline at end of file + envFrom: + - configMapKeyRef: + name: magic8ball-configmap \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/vars.sh b/scenarios/AksOpenAiTerraform/scripts/vars.sh deleted file mode 100644 index 4af9bf3b3..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/vars.sh +++ /dev/null @@ -1,10 +0,0 @@ -cd terraform - -RESOURCE_GROUP=$($(terraform output resource_group_name)) -LOCATION="westus3" -SUBSCRIPTION_ID=$(az account show --query id --output tsv) -TENANT_ID=$(az account show --query tenantId --output tsv) - -# Sample Application -namespace="magic8ball" -serviceAccountName="magic8ball-sa" \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index 7d0946ad0..2d237c210 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -20,6 +20,9 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { image_cleaner_enabled = true image_cleaner_interval_hours = 72 + workload_identity_enabled = true + oidc_issuer_enabled = true + default_node_pool { name = "system" node_count = 1 diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf index 80519e8d5..346acd0ca 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf @@ -7,7 +7,7 @@ output "id" { } output "aks_identity_principal_id" { - value = azurerm_user_assigned_identity.aks.id + value = azurerm_user_assigned_identity.aks.principal_id } output "kubelet_identity_object_id" { From de35561bbcf1d654988dcaa205f1b4a1fdf48dd3 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 25 Feb 2025 01:42:53 -0500 Subject: [PATCH 080/119] Fixes --- .../AksOpenAiTerraform/scripts/app/Dockerfile | 92 +---------- .../AksOpenAiTerraform/scripts/app/app.py | 47 +----- .../scripts/app/requirements.txt | 144 +----------------- .../AksOpenAiTerraform/scripts/deploy.sh | 15 +- .../terraform/.terraform.lock.hcl | 2 + .../AksOpenAiTerraform/terraform/main.tf | 8 +- .../terraform/modules/aks/main.tf | 6 +- .../terraform/modules/aks/outputs.tf | 8 +- .../AksOpenAiTerraform/terraform/outputs.tf | 6 +- 9 files changed, 36 insertions(+), 292 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/scripts/app/Dockerfile b/scenarios/AksOpenAiTerraform/scripts/app/Dockerfile index 2f603014f..0b9cb2035 100644 --- a/scenarios/AksOpenAiTerraform/scripts/app/Dockerfile +++ b/scenarios/AksOpenAiTerraform/scripts/app/Dockerfile @@ -1,94 +1,12 @@ -# app/Dockerfile - -# # Stage 1 - Install build dependencies - -# A Dockerfile must start with a FROM instruction which sets the base image for the container. -# The Python images come in many flavors, each designed for a specific use case. -# The python:3.11-slim image is a good base image for most applications. -# It is a minimal image built on top of Debian Linux and includes only the necessary packages to run Python. -# The slim image is a good choice because it is small and contains only the packages needed to run Python. -# For more information, see: -# * https://hub.docker.com/_/python -# * https://docs.streamlit.io/knowledge-base/tutorials/deploy/docker FROM python:3.11-slim AS builder - -# The WORKDIR instruction sets the working directory for any RUN, CMD, ENTRYPOINT, COPY and ADD instructions that follow it in the Dockerfile. -# If the WORKDIR doesn’t exist, it will be created even if it’s not used in any subsequent Dockerfile instruction. -# For more information, see: https://docs.docker.com/engine/reference/builder/#workdir -WORKDIR /app - -# Set environment variables. -# The ENV instruction sets the environment variable to the value . -# This value will be in the environment of all “descendant” Dockerfile commands and can be replaced inline in many as well. -# For more information, see: https://docs.docker.com/engine/reference/builder/#env -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 - -# Install git so that we can clone the app code from a remote repo using the RUN instruction. -# The RUN comand has 2 forms: -# * RUN (shell form, the command is run in a shell, which by default is /bin/sh -c on Linux or cmd /S /C on Windows) -# * RUN ["executable", "param1", "param2"] (exec form) -# The RUN instruction will execute any commands in a new layer on top of the current image and commit the results. -# The resulting committed image will be used for the next step in the Dockerfile. -# For more information, see: https://docs.docker.com/engine/reference/builder/#run -RUN apt-get update && apt-get install -y \ - build-essential \ - curl \ - software-properties-common \ - git \ - && rm -rf /var/lib/apt/lists/* - -# Create a virtualenv to keep dependencies together -RUN python -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" - -# Clone the requirements.txt which contains dependencies to WORKDIR -# COPY has two forms: -# * COPY (this copies the files from the local machine to the container's own filesystem) -# * COPY ["",... ""] (this form is required for paths containing whitespace) -# For more information, see: https://docs.docker.com/engine/reference/builder/#copy -COPY requirements.txt . - -# Install the Python dependencies -RUN pip install --no-cache-dir --no-deps -r requirements.txt - -# Stage 2 - Copy only necessary files to the runner stage - -# The FROM instruction initializes a new build stage for the application -FROM python:3.11-slim - -# Sets the working directory to /app WORKDIR /app -# Copy the virtual environment from the builder stage -COPY --from=builder /opt/venv /opt/venv +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 -# Set environment variables -ENV PATH="/opt/venv/bin:$PATH" +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt -# Clone the app.py containing the application code -COPY app.py . - -# Copy the images folder to WORKDIR -# The ADD instruction copies new files, directories or remote file URLs from and adds them to the filesystem of the image at the path . -# For more information, see: https://docs.docker.com/engine/reference/builder/#add -ADD images ./images - -# The EXPOSE instruction informs Docker that the container listens on the specified network ports at runtime. -# For more information, see: https://docs.docker.com/engine/reference/builder/#expose +COPY . . EXPOSE 8501 - -# The HEALTHCHECK instruction has two forms: -# * HEALTHCHECK [OPTIONS] CMD command (check container health by running a command inside the container) -# * HEALTHCHECK NONE (disable any healthcheck inherited from the base image) -# The HEALTHCHECK instruction tells Docker how to test a container to check that it is still working. -# This can detect cases such as a web server that is stuck in an infinite loop and unable to handle new connections, -# even though the server process is still running. For more information, see: https://docs.docker.com/engine/reference/builder/#healthcheck -HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health - -# The ENTRYPOINT instruction has two forms: -# * ENTRYPOINT ["executable", "param1", "param2"] (exec form, preferred) -# * ENTRYPOINT command param1 param2 (shell form) -# The ENTRYPOINT instruction allows you to configure a container that will run as an executable. -# For more information, see: https://docs.docker.com/engine/reference/builder/#entrypoint ENTRYPOINT ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"] \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/app/app.py b/scenarios/AksOpenAiTerraform/scripts/app/app.py index 4211c57ca..76fa07164 100644 --- a/scenarios/AksOpenAiTerraform/scripts/app/app.py +++ b/scenarios/AksOpenAiTerraform/scripts/app/app.py @@ -1,47 +1,13 @@ -""" -MIT License - -Copyright (c) 2023 Paolo Salvatori - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -# This sample is based on the following article: -# # - https://levelup.gitconnected.com/its-time-to-create-a-private-chatgpt-for-yourself-today-6503649e7bb6 # -# Use pip to install the following packages: -# -# - streamlit -# - openai -# - streamlit-chat -# - azure.identity -# - dotenv -# # Make sure to provide a value for the following environment variables: # -# - AZURE_OPENAI_BASE: the URL of your Azure OpenAI resource, for example https://eastus.api.cognitive.microsoft.com/ -# - AZURE_OPENAI_KEY: the key of your Azure OpenAI resource -# - AZURE_OPENAI_DEPLOYMENT: the name of the ChatGPT deployment used by your Azure OpenAI resource -# - AZURE_OPENAI_MODEL: the name of the ChatGPT model used by your Azure OpenAI resource, for example gpt-35-turbo -# - TITLE: the title of the Streamlit app -# - TEMPERATURE: the temperature used by the OpenAI API to generate the response +# - AZURE_OPENAI_BASE (ex: https://eastus.api.cognitive.microsoft.com/) +# - AZURE_OPENAI_KEY +# - AZURE_OPENAI_DEPLOYMENT +# - AZURE_OPENAI_MODEL +# - TITLE +# - TEMPERATURE # - SYSTEM: give the model instructions about how it should behave and any context it should reference when generating a response. # Used to describe the assistant's personality. # @@ -64,7 +30,6 @@ # # - streamlit run app.py -# Import packages import os import sys import time diff --git a/scenarios/AksOpenAiTerraform/scripts/app/requirements.txt b/scenarios/AksOpenAiTerraform/scripts/app/requirements.txt index 0278b846c..ec7c03c8b 100644 --- a/scenarios/AksOpenAiTerraform/scripts/app/requirements.txt +++ b/scenarios/AksOpenAiTerraform/scripts/app/requirements.txt @@ -1,145 +1,5 @@ -aiohttp==3.8.4 -aiosignal==1.3.1 -altair==4.2.2 -anyio==3.6.2 -async-timeout==4.0.2 -attrs==23.1.0 -autopep8==1.6.0 -azure-core==1.26.4 -azure-identity==1.13.0 -backoff==2.2.1 -blinker==1.6.2 -cachetools==5.3.0 -certifi==2021.10.8 -cffi==1.15.1 -charset-normalizer==2.0.7 -chromadb==0.3.22 -click==8.0.3 -clickhouse-connect==0.5.24 -cmake==3.26.3 -cryptography==40.0.2 -dataclasses-json==0.5.7 -debugpy==1.6.7 -decorator==5.1.1 -duckdb==0.7.1 -entrypoints==0.4 -et-xmlfile==1.1.0 -fastapi==0.95.1 -filelock==3.12.0 -Flask==2.0.2 -frozenlist==1.3.3 -fsspec==2023.5.0 -gitdb==4.0.10 -GitPython==3.1.31 -greenlet==2.0.2 -h11==0.14.0 -hnswlib==0.7.0 -httptools==0.5.0 -huggingface-hub==0.14.1 -idna==3.3 -importlib-metadata==6.6.0 -itsdangerous==2.0.1 -jc==1.23.1 -Jinja2==3.0.2 -joblib==1.2.0 -jsonschema==4.17.3 -langchain==0.0.169 -lit==16.0.3 -llama-index==0.6.8 -lz4==4.3.2 -markdown-it-py==2.2.0 -MarkupSafe==2.0.1 -marshmallow==3.19.0 -marshmallow-enum==1.5.1 -mdurl==0.1.2 -monotonic==1.6 -mpmath==1.3.0 -msal==1.22.0 -msal-extensions==1.0.0 -multidict==6.0.4 -mypy-extensions==1.0.0 -networkx==3.1 -nltk==3.8.1 -numexpr==2.8.4 -numpy==1.24.3 -nvidia-cublas-cu11==11.10.3.66 -nvidia-cuda-cupti-cu11==11.7.101 -nvidia-cuda-nvrtc-cu11==11.7.99 -nvidia-cuda-runtime-cu11==11.7.99 -nvidia-cudnn-cu11==8.5.0.96 -nvidia-cufft-cu11==10.9.0.58 -nvidia-curand-cu11==10.2.10.91 -nvidia-cusolver-cu11==11.4.0.1 -nvidia-cusparse-cu11==11.7.4.91 -nvidia-nccl-cu11==2.14.3 -nvidia-nvtx-cu11==11.7.91 -openai==0.27.7 -openapi-schema-pydantic==1.2.4 -openpyxl==3.0.9 -packaging==23.1 -pandas==2.0.1 -pandas-stubs==1.2.0.35 -Pillow==9.5.0 -pipdeptree==2.7.1 -portalocker==2.7.0 -posthog==3.0.1 -protobuf==3.20.3 -pyarrow==12.0.0 -pycodestyle==2.8.0 -pycparser==2.21 -pydantic==1.10.7 -pydeck==0.8.1b0 -Pygments==2.15.1 -PyJWT==2.7.0 -Pympler==1.0.1 -PyPDF2==3.0.1 -pyrsistent==0.19.3 -python-dateutil==2.8.2 python-dotenv==0.19.2 -pytz==2021.3 -PyYAML==6.0 -regex==2023.5.5 -requests==2.29.0 -rich==13.3.5 -ruamel.yaml==0.17.21 -ruamel.yaml.clib==0.2.7 -scikit-learn==1.2.2 -scipy==1.10.1 -sentence-transformers==2.2.2 -sentencepiece==0.1.99 -six==1.16.0 -smmap==5.0.0 -sniffio==1.3.0 -SQLAlchemy==2.0.13 -starlette==0.26.1 streamlit==1.22.0 streamlit-chat==0.0.2.2 -sympy==1.12 -tenacity==8.2.2 -threadpoolctl==3.1.0 -tiktoken==0.4.0 -tokenizers==0.13.3 -toml==0.10.2 -toolz==0.12.0 -torch==2.0.1 -torchvision==0.15.2 -tornado==6.3.2 -tqdm==4.62.3 -transformers==4.29.1 -triton==2.0.0 -typing-inspect==0.8.0 -typing_extensions==4.5.0 -tzdata==2023.3 -tzlocal==5.0.1 -urllib3==1.26.7 -uvicorn==0.22.0 -uvloop==0.17.0 -validators==0.20.0 -watchdog==3.0.0 -watchfiles==0.19.0 -websockets==11.0.3 -Werkzeug==2.0.2 -xmltodict==0.13.0 -yarl==1.9.2 -zipp==3.15.0 -zstandard==0.21.0 +azure-identity==1.13.0 +openai==0.27.7 \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/deploy.sh b/scenarios/AksOpenAiTerraform/scripts/deploy.sh index b60748fa8..ad2b7615d 100644 --- a/scenarios/AksOpenAiTerraform/scripts/deploy.sh +++ b/scenarios/AksOpenAiTerraform/scripts/deploy.sh @@ -1,16 +1,16 @@ -# Variables +#!/bin/bash + SUBSCRIPTION_ID=$(az account show --query id --output tsv) TENANT_ID=$(az account show --query tenantId --output tsv) RESOURCE_GROUP=$(terraform output resource_group_name) LOCATION="westus3" -# Build/Push App's Docker image -ACR_NAME=$(terraform output resource_group_name) +# Build Image +ACR_NAME=$(terraform output acr_name) az acr login --name $ACR_NAME ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) -docker build -t $ACR_URL/$ACR_NAME.azurecr.io/magic8ball:v1 ./app --push +docker build -t $ACR_URL/magic8ball:v1 ./app --push -# Get AKS credentials az aks get-credentials \ --admin \ --name $clusterName \ @@ -18,11 +18,6 @@ az aks get-credentials \ --subscription $subscriptionId \ --only-show-errors -# Install Helm -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 -o get_helm.sh -s -chmod 700 get_helm.sh -./get_helm.sh &>/dev/null - # Install NGINX ingress controller helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx helm install nginx-ingress ingress-nginx/ingress-nginx \ diff --git a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl index 2aeb47adf..8faadd9c3 100644 --- a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl +++ b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl @@ -5,6 +5,7 @@ provider "registry.terraform.io/hashicorp/azurerm" { version = "4.16.0" constraints = "~> 4.16.0" hashes = [ + "h1:7e25Wr4cpUvlAcwL+9ZOeeA1xha84LqTZNviDaVQFlo=", "h1:UNZga7kYMfYfDHmuP6LvHmJNXlb3fyvRY1tA9ol6yY4=", "zh:2035e461a94bd4180557a06f8e56f228a8a035608d0dac4d08e5870cf9265276", "zh:3f15778a22ef1b9d0fa28670e5ea6ef1094b0be2533f43f350a2ef15d471b353", @@ -24,6 +25,7 @@ provider "registry.terraform.io/hashicorp/azurerm" { provider "registry.terraform.io/hashicorp/random" { version = "3.6.3" hashes = [ + "h1:Fnaec9vA8sZ8BXVlN3Xn9Jz3zghSETIKg7ch8oXhxno=", "h1:N2IQabOiZC5eCEGrfgVS6ChVmRDh1ENtfHgGjnV4QQQ=", "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8", diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 80d57ec04..e2298c97c 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -62,7 +62,7 @@ module "openai" { log_analytics_workspace_id = module.log_analytics_workspace.id } -module "aks_cluster" { +module "aks" { source = "./modules/aks" name = "AksCluster" location = var.location @@ -252,7 +252,7 @@ resource "azurerm_federated_identity_credential" "this" { resource_group_name = azurerm_resource_group.main.name audience = ["api://AzureADTokenExchange"] - issuer = module.aks_cluster.oidc_issuer_url + issuer = module.aks.oidc_issuer_url parent_id = azurerm_user_assigned_identity.aks_workload.id subject = "system:serviceaccount:${local.namespace}:${local.service_account_name}" } @@ -266,11 +266,11 @@ resource "azurerm_role_assignment" "cognitive_services_user_assignment" { resource "azurerm_role_assignment" "network_contributor_assignment" { role_definition_name = "Network Contributor" scope = azurerm_resource_group.main.id - principal_id = module.aks_cluster.aks_identity_principal_id + principal_id = module.aks.aks_identity_principal_id } resource "azurerm_role_assignment" "acr_pull_assignment" { role_definition_name = "AcrPull" scope = module.container_registry.id - principal_id = module.aks_cluster.kubelet_identity_object_id + principal_id = module.aks.kubelet_identity_object_id } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index 2d237c210..fb069a07a 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -8,7 +8,7 @@ resource "azurerm_user_assigned_identity" "aks" { location = var.location } -resource "azurerm_kubernetes_cluster" "aks_cluster" { +resource "azurerm_kubernetes_cluster" "main" { name = var.name location = var.location resource_group_name = var.resource_group_name @@ -56,7 +56,7 @@ resource "azurerm_kubernetes_cluster" "aks_cluster" { } resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { - kubernetes_cluster_id = azurerm_kubernetes_cluster.aks_cluster.id + kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id name = "user" vm_size = var.user_node_pool_vm_size mode = "User" @@ -70,7 +70,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { resource "azurerm_monitor_diagnostic_setting" "settings" { name = "AksDiagnosticsSettings" - target_resource_id = azurerm_kubernetes_cluster.aks_cluster.id + target_resource_id = azurerm_kubernetes_cluster.main.id log_analytics_workspace_id = var.log_analytics_workspace_id enabled_log { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf index 346acd0ca..158f62992 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf @@ -1,9 +1,9 @@ output "name" { - value = azurerm_kubernetes_cluster.aks_cluster.name + value = azurerm_kubernetes_cluster.main.name } output "id" { - value = azurerm_kubernetes_cluster.aks_cluster.id + value = azurerm_kubernetes_cluster.main.id } output "aks_identity_principal_id" { @@ -11,9 +11,9 @@ output "aks_identity_principal_id" { } output "kubelet_identity_object_id" { - value = azurerm_kubernetes_cluster.aks_cluster.kubelet_identity.0.object_id + value = azurerm_kubernetes_cluster.main.kubelet_identity.0.object_id } output "oidc_issuer_url" { - value = azurerm_kubernetes_cluster.aks_cluster.oidc_issuer_url + value = azurerm_kubernetes_cluster.main.oidc_issuer_url } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf index 9f0f3f4bf..d00f3ca0d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/outputs.tf @@ -2,6 +2,10 @@ output "resource_group_name" { value = azurerm_resource_group.main.name } -output "acr_url" { +output "cluster_name" { + value = module.aks.name +} + +output "acr_name" { value = module.container_registry.name } \ No newline at end of file From 9d8645a42a051b96602c3301b69ba2901015851e Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 25 Feb 2025 02:28:31 -0500 Subject: [PATCH 081/119] Move --- scenarios/AksOpenAiTerraform/README.md | 3 +- .../{scripts => }/app/Dockerfile | 0 .../{scripts => }/app/app.py | 24 ++- .../{scripts => }/app/images/magic8ball.png | Bin .../{scripts => }/app/images/robot.png | Bin .../{scripts => }/app/requirements.txt | 0 .../{scripts => }/deploy.sh | 12 +- .../AksOpenAiTerraform/quickstart-app.yml | 169 ++++++++++++++++++ .../scripts/manifests/cluster-issuer.yml | 18 -- .../scripts/manifests/config-map.yml | 14 -- .../scripts/manifests/deployment.yml | 80 --------- .../scripts/manifests/ingress.yml | 30 ---- .../scripts/manifests/service-account.yml | 10 -- .../scripts/manifests/service.yml | 13 -- .../scripts/register-preview-features.sh | 25 --- .../terraform/modules/aks/main.tf | 12 -- 16 files changed, 188 insertions(+), 222 deletions(-) rename scenarios/AksOpenAiTerraform/{scripts => }/app/Dockerfile (100%) rename scenarios/AksOpenAiTerraform/{scripts => }/app/app.py (93%) rename scenarios/AksOpenAiTerraform/{scripts => }/app/images/magic8ball.png (100%) rename scenarios/AksOpenAiTerraform/{scripts => }/app/images/robot.png (100%) rename scenarios/AksOpenAiTerraform/{scripts => }/app/requirements.txt (100%) rename scenarios/AksOpenAiTerraform/{scripts => }/deploy.sh (93%) create mode 100644 scenarios/AksOpenAiTerraform/quickstart-app.yml delete mode 100644 scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml delete mode 100644 scenarios/AksOpenAiTerraform/scripts/manifests/config-map.yml delete mode 100644 scenarios/AksOpenAiTerraform/scripts/manifests/deployment.yml delete mode 100644 scenarios/AksOpenAiTerraform/scripts/manifests/ingress.yml delete mode 100644 scenarios/AksOpenAiTerraform/scripts/manifests/service-account.yml delete mode 100644 scenarios/AksOpenAiTerraform/scripts/manifests/service.yml delete mode 100644 scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index d81e304e9..8497390a3 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -13,7 +13,8 @@ ms.custom: innovation-engine, linux-related-content Run commands below to set up AKS extensions for Azure. ```bash -./scripts/register-preview-features.sh +az extension add --name aks-preview +az aks install-cli ``` ## Set up Subscription ID to authenticate for Terraform diff --git a/scenarios/AksOpenAiTerraform/scripts/app/Dockerfile b/scenarios/AksOpenAiTerraform/app/Dockerfile similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/app/Dockerfile rename to scenarios/AksOpenAiTerraform/app/Dockerfile diff --git a/scenarios/AksOpenAiTerraform/scripts/app/app.py b/scenarios/AksOpenAiTerraform/app/app.py similarity index 93% rename from scenarios/AksOpenAiTerraform/scripts/app/app.py rename to scenarios/AksOpenAiTerraform/app/app.py index 76fa07164..bcca70af9 100644 --- a/scenarios/AksOpenAiTerraform/scripts/app/app.py +++ b/scenarios/AksOpenAiTerraform/app/app.py @@ -1,22 +1,21 @@ -# - https://levelup.gitconnected.com/its-time-to-create-a-private-chatgpt-for-yourself-today-6503649e7bb6 +# https://levelup.gitconnected.com/its-time-to-create-a-private-chatgpt-for-yourself-today-6503649e7bb6 # # Make sure to provide a value for the following environment variables: -# -# - AZURE_OPENAI_BASE (ex: https://eastus.api.cognitive.microsoft.com/) -# - AZURE_OPENAI_KEY -# - AZURE_OPENAI_DEPLOYMENT -# - AZURE_OPENAI_MODEL -# - TITLE -# - TEMPERATURE -# - SYSTEM: give the model instructions about how it should behave and any context it should reference when generating a response. -# Used to describe the assistant's personality. +# - AZURE_OPENAI_BASE (ex: https://eastus.api.cognitive.microsoft.com/) +# - AZURE_OPENAI_KEY +# - AZURE_OPENAI_DEPLOYMENT +# - AZURE_OPENAI_MODEL +# - TITLE +# - TEMPERATURE +# - SYSTEM (Used to describe the assistant's personality.) # # You can use two different authentication methods: # -# - API key: set the AZURE_OPENAI_TYPE environment variable to azure and the AZURE_OPENAI_KEY environment variable to the key of +# - API key: set the AZURE_OPENAI_TYPE environment variable to azure and the AZURE_OPENAI_KEY environment variable to the key of # your Azure OpenAI resource. You can use the regional endpoint, such as https://eastus.api.cognitive.microsoft.com/, passed in # the AZURE_OPENAI_BASE environment variable, to connect to the Azure OpenAI resource. -# - Azure Active Directory: set the AZURE_OPENAI_TYPE environment variable to azure_ad and use a service principal or managed +# +# - Azure Active Directory: set the AZURE_OPENAI_TYPE environment variable to azure_ad and use a service principal or managed # identity with the DefaultAzureCredential object to acquire a token. For more information on the DefaultAzureCredential in Python, # see https://docs.microsoft.com/en-us/azure/developer/python/azure-sdk-authenticate?tabs=cmd # Make sure to assign the "Cognitive Services User" role to the service principal or managed identity used to authenticate to @@ -27,7 +26,6 @@ # Hence, make sure to pass the endpoint containing the custom domain in the AZURE_OPENAI_BASE environment variable. # # Use the following command to run the app: -# # - streamlit run app.py import os diff --git a/scenarios/AksOpenAiTerraform/scripts/app/images/magic8ball.png b/scenarios/AksOpenAiTerraform/app/images/magic8ball.png similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/app/images/magic8ball.png rename to scenarios/AksOpenAiTerraform/app/images/magic8ball.png diff --git a/scenarios/AksOpenAiTerraform/scripts/app/images/robot.png b/scenarios/AksOpenAiTerraform/app/images/robot.png similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/app/images/robot.png rename to scenarios/AksOpenAiTerraform/app/images/robot.png diff --git a/scenarios/AksOpenAiTerraform/scripts/app/requirements.txt b/scenarios/AksOpenAiTerraform/app/requirements.txt similarity index 100% rename from scenarios/AksOpenAiTerraform/scripts/app/requirements.txt rename to scenarios/AksOpenAiTerraform/app/requirements.txt diff --git a/scenarios/AksOpenAiTerraform/scripts/deploy.sh b/scenarios/AksOpenAiTerraform/deploy.sh similarity index 93% rename from scenarios/AksOpenAiTerraform/scripts/deploy.sh rename to scenarios/AksOpenAiTerraform/deploy.sh index ad2b7615d..bd8d5f159 100644 --- a/scenarios/AksOpenAiTerraform/scripts/deploy.sh +++ b/scenarios/AksOpenAiTerraform/deploy.sh @@ -4,19 +4,19 @@ SUBSCRIPTION_ID=$(az account show --query id --output tsv) TENANT_ID=$(az account show --query tenantId --output tsv) RESOURCE_GROUP=$(terraform output resource_group_name) LOCATION="westus3" +CLUSTER_NAME=$(terraform output cluster_name) # Build Image -ACR_NAME=$(terraform output acr_name) +ACR_NAME="$(terraform output acr_name)" az acr login --name $ACR_NAME ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) docker build -t $ACR_URL/magic8ball:v1 ./app --push az aks get-credentials \ --admin \ - --name $clusterName \ - --resource-group $resourceGroupName \ - --subscription $subscriptionId \ - --only-show-errors + --name $CLUSTER_NAME \ + --resource-group $RESOURCE_GROUP \ + --subscription $SUBSCRIPTION_ID \ # Install NGINX ingress controller helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx @@ -29,7 +29,7 @@ helm install nginx-ingress ingress-nginx/ingress-nginx \ --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz \ --set controller.metrics.enabled=true \ --set controller.metrics.serviceMonitor.enabled=true \ - --set controller.metrics.serviceMonitor.additionalLabels.release="prometheus" \ + --set controller.metrics.serviceMonitor.additionalLabels.release="prometheus" # Install Cert manager helm repo add jetstack https://charts.jetstack.io diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml new file mode 100644 index 000000000..af4eab8e7 --- /dev/null +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -0,0 +1,169 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: magic8ball-configmap +data: + TITLE: "Magic 8 Ball" + LABEL: "Pose your question and cross your fingers!" + TEMPERATURE: "0.9" + IMAGE_WIDTH: "80" + AZURE_OPENAI_TYPE: azure_ad + AZURE_OPENAI_BASE: https://myopenai.openai.azure.com/ + AZURE_OPENAI_MODEL: gpt-35-turbo + AZURE_OPENAI_DEPLOYMENT: magic8ballGPT +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: magic8ball + labels: + app: magic8ball +spec: + replicas: 3 + selector: + matchLabels: + app: magic8ball + azure.workload.identity/use: "true" + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + minReadySeconds: 5 + template: + metadata: + labels: + app: magic8ball + azure.workload.identity/use: "true" + prometheus.io/scrape: "true" + spec: + serviceAccountName: magic8ball-sa + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: magic8ball + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: magic8ball + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: magic8ball + image: paolosalvatori.azurecr.io/magic8ball:v1 + imagePullPolicy: Always + resources: + requests: + memory: "128Mi" + cpu: "250m" + limits: + memory: "256Mi" + cpu: "500m" + ports: + - containerPort: 8501 + livenessProbe: + httpGet: + path: / + port: 8501 + failureThreshold: 1 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 8501 + failureThreshold: 1 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + startupProbe: + httpGet: + path: / + port: 8501 + failureThreshold: 1 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + envFrom: + - configMapKeyRef: + name: magic8ball-configmap +--- +apiVersion: v1 +kind: Service +metadata: + name: magic8ball + labels: + app: magic8ball +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 8501 + selector: + app: magic8ball +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + azure.workload.identity/client-id: $workloadManagedIdentityClientId + azure.workload.identity/tenant-id: $tenantId + labels: + azure.workload.identity/use: "true" + name: $serviceAccountName + namespace: $namespace +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: magic8ball-ingress + annotations: + cert-manager.io/cluster-issuer: letsencrypt-nginx + cert-manager.io/acme-challenge-type: http01 + nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" + nginx.ingress.kubernetes.io/proxy-send-timeout: "360" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + nginx.ingress.kubernetes.io/proxy-next-upstream-timeout: "360" + nginx.ingress.kubernetes.io/configuration-snippet: | + more_set_headers "X-Frame-Options: SAMEORIGIN"; +spec: + ingressClassName: nginx + tls: + - hosts: + - magic8ball.contoso.com + secretName: tls-secret + rules: + - host: magic8ball.contoso.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: magic8ball + port: + number: 8501 +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-nginx +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: {{ .Values.email }} + privateKeySecretRef: + name: letsencrypt + solvers: + - http01: + ingress: + class: nginx + podTemplate: + spec: + nodeSelector: + "kubernetes.io/os": linux \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml deleted file mode 100644 index 6cc55451f..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/manifests/cluster-issuer.yml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer -metadata: - name: letsencrypt-nginx -spec: - acme: - server: https://acme-v02.api.letsencrypt.org/directory - email: {{ .Values.email }} - privateKeySecretRef: - name: letsencrypt - solvers: - - http01: - ingress: - class: nginx - podTemplate: - spec: - nodeSelector: - "kubernetes.io/os": linux \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/config-map.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/config-map.yml deleted file mode 100644 index fb668c832..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/manifests/config-map.yml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: magic8ball-configmap -data: - TITLE: "Magic 8 Ball" - LABEL: "Pose your question and cross your fingers!" - TEMPERATURE: "0.9" - IMAGE_WIDTH: "80" - AZURE_OPENAI_TYPE: azure_ad - AZURE_OPENAI_BASE: https://myopenai.openai.azure.com/ - AZURE_OPENAI_KEY: "" - AZURE_OPENAI_MODEL: gpt-35-turbo - AZURE_OPENAI_DEPLOYMENT: magic8ballGPT diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/deployment.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/deployment.yml deleted file mode 100644 index ee805c4aa..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/manifests/deployment.yml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: magic8ball - labels: - app: magic8ball -spec: - replicas: 3 - selector: - matchLabels: - app: magic8ball - azure.workload.identity/use: "true" - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - minReadySeconds: 5 - template: - metadata: - labels: - app: magic8ball - azure.workload.identity/use: "true" - prometheus.io/scrape: "true" - spec: - serviceAccountName: magic8ball-sa - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - app: magic8ball - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - app: magic8ball - nodeSelector: - "kubernetes.io/os": linux - containers: - - name: magic8ball - image: paolosalvatori.azurecr.io/magic8ball:v1 - imagePullPolicy: Always - resources: - requests: - memory: "128Mi" - cpu: "250m" - limits: - memory: "256Mi" - cpu: "500m" - ports: - - containerPort: 8501 - livenessProbe: - httpGet: - path: / - port: 8501 - failureThreshold: 1 - initialDelaySeconds: 60 - periodSeconds: 30 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: / - port: 8501 - failureThreshold: 1 - initialDelaySeconds: 60 - periodSeconds: 30 - timeoutSeconds: 5 - startupProbe: - httpGet: - path: / - port: 8501 - failureThreshold: 1 - initialDelaySeconds: 60 - periodSeconds: 30 - timeoutSeconds: 5 - envFrom: - - configMapKeyRef: - name: magic8ball-configmap \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/ingress.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/ingress.yml deleted file mode 100644 index 2e56a46d4..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/manifests/ingress.yml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: magic8ball-ingress - annotations: - cert-manager.io/cluster-issuer: letsencrypt-nginx - cert-manager.io/acme-challenge-type: http01 - nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" - nginx.ingress.kubernetes.io/proxy-send-timeout: "360" - nginx.ingress.kubernetes.io/proxy-read-timeout: "360" - nginx.ingress.kubernetes.io/proxy-next-upstream-timeout: "360" - nginx.ingress.kubernetes.io/configuration-snippet: | - more_set_headers "X-Frame-Options: SAMEORIGIN"; -spec: - ingressClassName: nginx - tls: - - hosts: - - magic8ball.contoso.com - secretName: tls-secret - rules: - - host: magic8ball.contoso.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: magic8ball - port: - number: 8501 \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/service-account.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/service-account.yml deleted file mode 100644 index a5ab35826..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/manifests/service-account.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - annotations: - azure.workload.identity/client-id: $workloadManagedIdentityClientId - azure.workload.identity/tenant-id: $tenantId - labels: - azure.workload.identity/use: "true" - name: $serviceAccountName - namespace: $namespace \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/scripts/manifests/service.yml b/scenarios/AksOpenAiTerraform/scripts/manifests/service.yml deleted file mode 100644 index c6c92c3ef..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/manifests/service.yml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: magic8ball - labels: - app: magic8ball -spec: - type: ClusterIP - ports: - - protocol: TCP - port: 8501 - selector: - app: magic8ball diff --git a/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh b/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh deleted file mode 100644 index 2abdce2a7..000000000 --- a/scenarios/AksOpenAiTerraform/scripts/register-preview-features.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Install aks-preview Azure extension -echo "Checking if [aks-preview] extension is already installed..." -az extension show --name aks-preview - -if [[ $? == 0 ]]; then - echo "[aks-preview] extension is already installed" - - # Update the extension to make sure you have the latest version installed - echo "Updating [aks-preview] extension..." - az extension update --name aks-preview -else - echo "[aks-preview] extension is not installed. Installing..." - - # Install aks-preview extension - az extension add --name aks-preview 1>/dev/null - - if [[ $? == 0 ]]; then - echo "[aks-preview] extension successfully installed" - else - echo "Failed to install [aks-preview] extension" - exit - fi -fi \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index fb069a07a..dd75b5f2e 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -37,22 +37,10 @@ resource "azurerm_kubernetes_cluster" "main" { identity_ids = tolist([azurerm_user_assigned_identity.aks.id]) } - network_profile { - dns_service_ip = "10.2.0.10" - network_plugin = "azure" - outbound_type = "userAssignedNATGateway" - service_cidr = "10.2.0.0/24" - } - oms_agent { msi_auth_for_monitoring_enabled = true log_analytics_workspace_id = var.log_analytics_workspace_id } - - azure_active_directory_role_based_access_control { - tenant_id = var.tenant_id - azure_rbac_enabled = true - } } resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { From 2d99179428715bc41374a25707dff4d898ac42ff Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 25 Feb 2025 14:27:03 -0500 Subject: [PATCH 082/119] Add back network profile --- scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index dd75b5f2e..3b86dc55a 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -37,6 +37,13 @@ resource "azurerm_kubernetes_cluster" "main" { identity_ids = tolist([azurerm_user_assigned_identity.aks.id]) } + network_profile { + dns_service_ip = "10.2.0.10" + network_plugin = "azure" + outbound_type = "userAssignedNATGateway" + service_cidr = "10.2.0.0/24" + } + oms_agent { msi_auth_for_monitoring_enabled = true log_analytics_workspace_id = var.log_analytics_workspace_id From c2c6fe82cf3104838e5def324e67c91616659dab Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 25 Feb 2025 17:33:36 -0500 Subject: [PATCH 083/119] WIP --- scenarios/AksOpenAiTerraform/app/Dockerfile | 2 +- scenarios/AksOpenAiTerraform/app/app.py | 2 +- .../app/{images => icons}/magic8ball.png | Bin .../app/{images => icons}/robot.png | Bin scenarios/AksOpenAiTerraform/deploy.sh | 66 +++++------ .../AksOpenAiTerraform/quickstart-app.yml | 111 ++++-------------- 6 files changed, 53 insertions(+), 128 deletions(-) rename scenarios/AksOpenAiTerraform/app/{images => icons}/magic8ball.png (100%) rename scenarios/AksOpenAiTerraform/app/{images => icons}/robot.png (100%) diff --git a/scenarios/AksOpenAiTerraform/app/Dockerfile b/scenarios/AksOpenAiTerraform/app/Dockerfile index 0b9cb2035..68dcce690 100644 --- a/scenarios/AksOpenAiTerraform/app/Dockerfile +++ b/scenarios/AksOpenAiTerraform/app/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim AS builder +FROM python:3.11-slim WORKDIR /app ENV PYTHONDONTWRITEBYTECODE=1 diff --git a/scenarios/AksOpenAiTerraform/app/app.py b/scenarios/AksOpenAiTerraform/app/app.py index bcca70af9..ee012133c 100644 --- a/scenarios/AksOpenAiTerraform/app/app.py +++ b/scenarios/AksOpenAiTerraform/app/app.py @@ -273,7 +273,7 @@ def user_change(): # Display the robot image with col1: - st.image(image = os.path.join("images", image_file_name), width = image_width) + st.image(image = os.path.join("icons", image_file_name), width = image_width) # Display the title with col2: diff --git a/scenarios/AksOpenAiTerraform/app/images/magic8ball.png b/scenarios/AksOpenAiTerraform/app/icons/magic8ball.png similarity index 100% rename from scenarios/AksOpenAiTerraform/app/images/magic8ball.png rename to scenarios/AksOpenAiTerraform/app/icons/magic8ball.png diff --git a/scenarios/AksOpenAiTerraform/app/images/robot.png b/scenarios/AksOpenAiTerraform/app/icons/robot.png similarity index 100% rename from scenarios/AksOpenAiTerraform/app/images/robot.png rename to scenarios/AksOpenAiTerraform/app/icons/robot.png diff --git a/scenarios/AksOpenAiTerraform/deploy.sh b/scenarios/AksOpenAiTerraform/deploy.sh index bd8d5f159..5cc9d8c04 100644 --- a/scenarios/AksOpenAiTerraform/deploy.sh +++ b/scenarios/AksOpenAiTerraform/deploy.sh @@ -1,70 +1,60 @@ #!/bin/bash +cd terraform SUBSCRIPTION_ID=$(az account show --query id --output tsv) -TENANT_ID=$(az account show --query tenantId --output tsv) -RESOURCE_GROUP=$(terraform output resource_group_name) -LOCATION="westus3" -CLUSTER_NAME=$(terraform output cluster_name) +RESOURCE_GROUP=$(terraform output -raw resource_group_name) +CLUSTER_NAME=$(terraform output -raw cluster_name) +ACR_NAME="$(terraform output -raw acr_name)" +# EMAIL="$(terraform output -raw email)" +EMAIL=ariaamini@microsoft.com +cd .. # Build Image -ACR_NAME="$(terraform output acr_name)" az acr login --name $ACR_NAME ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) -docker build -t $ACR_URL/magic8ball:v1 ./app --push +IMAGE=$ACR_URL/magic8ball:v1 +docker build -t $IMAGE ./app --push -az aks get-credentials \ - --admin \ - --name $CLUSTER_NAME \ - --resource-group $RESOURCE_GROUP \ - --subscription $SUBSCRIPTION_ID \ +# Login +az aks get-credentials --admin --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --subscription $SUBSCRIPTION_ID -# Install NGINX ingress controller +# Install Deps helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx -helm install nginx-ingress ingress-nginx/ingress-nginx \ +helm repo add jetstack https://charts.jetstack.io +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +# NGINX ingress controller +helm install ingress-nginx ingress-nginx/ingress-nginx \ --create-namespace \ --namespace "ingress-basic" \ - --set controller.replicaCount=3 \ + --set controller.replicaCount=2 \ --set controller.nodeSelector."kubernetes\.io/os"=linux \ --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz \ --set controller.metrics.enabled=true \ --set controller.metrics.serviceMonitor.enabled=true \ --set controller.metrics.serviceMonitor.additionalLabels.release="prometheus" - -# Install Cert manager -helm repo add jetstack https://charts.jetstack.io +# Cert manager helm install cert-manager jetstack/cert-manager \ --create-namespace \ - --namespace "cert-manager" \ - --set installCRDs=true \ + --namespace cert-manager \ + --set crds.enabled=true \ --set nodeSelector."kubernetes\.io/os"=linux - -# Install Prometheus -helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +# Prometheus helm install prometheus prometheus-community/kube-prometheus-stack \ --create-namespace \ --namespace prometheus \ --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false -NAMESPACE="magic8ball" -kubectl create namespace $NAMESPACE -kubectl apply -f cluster-issuer.yml -kubectl apply -f service-account.yml -kubectl apply -n $NAMESPACE -f ingress.yml -kubectl apply -n $NAMESPACE -f config-map.yml -kubectl apply -n $NAMESPACE -f deployment.yml -kubectl apply -f "service.yml" -n $NAMESPACE +export IMAGE +export EMAIL +echo $IMAGE +kubectl create namespace magic8ball +envsubst < quickstart-app.yml | kubectl apply -f - # Add DNS Record -ingressName="magic8ball-ingress" -publicIpAddress=$(kubectl get ingress $ingressName -n $namespace -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -if [ -n $publicIpAddress ]; then - echo "[$publicIpAddress] external IP address of the application gateway ingress controller successfully retrieved from the [$ingressName] ingress" -else - echo "Failed to retrieve the external IP address of the application gateway ingress controller from the [$ingressName] ingress" - exit -fi +publicIpAddress=$(kubectl get ingress magic8ball-ingress -n magic8ball -o jsonpath='{.status.loadBalancer.ingress[0].ip}') az network dns record-set a add-record \ --zone-name "contoso.com" \ --resource-group $RESOURCE_GROUP \ diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index af4eab8e7..8a9f40801 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -2,6 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: magic8ball-configmap + namespace: magic8ball data: TITLE: "Magic 8 Ball" LABEL: "Pose your question and cross your fingers!" @@ -16,121 +17,58 @@ apiVersion: apps/v1 kind: Deployment metadata: name: magic8ball + namespace: magic8ball labels: - app: magic8ball + app.kubernetes.io/name: magic8ball + azure.workload.identity/use: "true" spec: replicas: 3 selector: matchLabels: - app: magic8ball - azure.workload.identity/use: "true" - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - minReadySeconds: 5 + app.kubernetes.io/name: magic8ball template: metadata: labels: - app: magic8ball - azure.workload.identity/use: "true" - prometheus.io/scrape: "true" + app.kubernetes.io/name: magic8ball spec: - serviceAccountName: magic8ball-sa - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - app: magic8ball - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - app: magic8ball - nodeSelector: - "kubernetes.io/os": linux containers: - name: magic8ball - image: paolosalvatori.azurecr.io/magic8ball:v1 - imagePullPolicy: Always - resources: - requests: - memory: "128Mi" - cpu: "250m" - limits: - memory: "256Mi" - cpu: "500m" + image: $IMAGE ports: - containerPort: 8501 - livenessProbe: - httpGet: - path: / - port: 8501 - failureThreshold: 1 - initialDelaySeconds: 60 - periodSeconds: 30 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: / - port: 8501 - failureThreshold: 1 - initialDelaySeconds: 60 - periodSeconds: 30 - timeoutSeconds: 5 - startupProbe: - httpGet: - path: / - port: 8501 - failureThreshold: 1 - initialDelaySeconds: 60 - periodSeconds: 30 - timeoutSeconds: 5 envFrom: - - configMapKeyRef: + - configMapRef: name: magic8ball-configmap --- apiVersion: v1 kind: Service metadata: - name: magic8ball - labels: - app: magic8ball + name: magic8ball-service + namespace: magic8ball spec: + selector: + app.kubernetes.io/name: magic8ball type: ClusterIP ports: - protocol: TCP port: 8501 - selector: - app: magic8ball + targetPort: 8501 --- apiVersion: v1 kind: ServiceAccount metadata: + name: magic8ball-service-account + namespace: magic8ball annotations: - azure.workload.identity/client-id: $workloadManagedIdentityClientId - azure.workload.identity/tenant-id: $tenantId - labels: - azure.workload.identity/use: "true" - name: $serviceAccountName - namespace: $namespace + azure.workload.identity/client-id: $WORKLOAD_MANAGED_IDENTITY_CLIENT_ID --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: magic8ball-ingress + namespace: magic8ball annotations: - cert-manager.io/cluster-issuer: letsencrypt-nginx - cert-manager.io/acme-challenge-type: http01 - nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" - nginx.ingress.kubernetes.io/proxy-send-timeout: "360" - nginx.ingress.kubernetes.io/proxy-read-timeout: "360" - nginx.ingress.kubernetes.io/proxy-next-upstream-timeout: "360" - nginx.ingress.kubernetes.io/configuration-snippet: | - more_set_headers "X-Frame-Options: SAMEORIGIN"; + cert-manager.io/cluster-issuer: letsencrypt-dev spec: ingressClassName: nginx tls: @@ -152,18 +90,15 @@ spec: apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: - name: letsencrypt-nginx + name: letsencrypt-dev + namespace: cert-manager spec: acme: server: https://acme-v02.api.letsencrypt.org/directory - email: {{ .Values.email }} + email: $EMAIL privateKeySecretRef: - name: letsencrypt + name: tls-secret solvers: - http01: ingress: - class: nginx - podTemplate: - spec: - nodeSelector: - "kubernetes.io/os": linux \ No newline at end of file + ingressClassName: nginx \ No newline at end of file From e41428b627435570993e49d3385b0c5d1154944a Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 25 Feb 2025 18:11:24 -0500 Subject: [PATCH 084/119] Fix --- scenarios/AksOpenAiTerraform/deploy.sh | 23 ++++++++----------- .../AksOpenAiTerraform/quickstart-app.yml | 2 +- .../terraform/modules/aks/main.tf | 6 +++++ .../AksOpenAiTerraform/terraform/outputs.tf | 4 ++++ .../AksOpenAiTerraform/terraform/variables.tf | 5 ---- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/deploy.sh b/scenarios/AksOpenAiTerraform/deploy.sh index 5cc9d8c04..44239d32a 100644 --- a/scenarios/AksOpenAiTerraform/deploy.sh +++ b/scenarios/AksOpenAiTerraform/deploy.sh @@ -1,18 +1,18 @@ #!/bin/bash cd terraform -SUBSCRIPTION_ID=$(az account show --query id --output tsv) -RESOURCE_GROUP=$(terraform output -raw resource_group_name) -CLUSTER_NAME=$(terraform output -raw cluster_name) -ACR_NAME="$(terraform output -raw acr_name)" -# EMAIL="$(terraform output -raw email)" -EMAIL=ariaamini@microsoft.com +export RESOURCE_GROUP=$(terraform output -raw resource_group_name) +export CLUSTER_NAME=$(terraform output -raw cluster_name) +export WORKLOAD_MANAGED_IDENTITY_CLIENT_ID=$(terraform output -raw workload_managed_identity_client_id) +export ACR_NAME=$(terraform output -raw acr_name) cd .. +export SUBSCRIPTION_ID=$(az account show --query id --output tsv) +export EMAIL="amini5454@gmail.com" # Build Image az acr login --name $ACR_NAME -ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) -IMAGE=$ACR_URL/magic8ball:v1 +export ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) +export IMAGE=$ACR_URL/magic8ball:v1 docker build -t $IMAGE ./app --push # Login @@ -47,16 +47,13 @@ helm install prometheus prometheus-community/kube-prometheus-stack \ --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false -export IMAGE -export EMAIL -echo $IMAGE kubectl create namespace magic8ball envsubst < quickstart-app.yml | kubectl apply -f - # Add DNS Record -publicIpAddress=$(kubectl get ingress magic8ball-ingress -n magic8ball -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +PUBLIC_IP=$(kubectl get ingress magic8ball-ingress -n magic8ball -o jsonpath='{.status.loadBalancer.ingress[0].ip}') az network dns record-set a add-record \ --zone-name "contoso.com" \ --resource-group $RESOURCE_GROUP \ --record-set-name magic8ball \ - --ipv4-address $publicIpAddress \ No newline at end of file + --ipv4-address $PUBLIC_IP \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index 8a9f40801..19d67ba2a 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -66,7 +66,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: magic8ball-ingress - namespace: magic8ball + namespace: ingress-nginx annotations: cert-manager.io/cluster-issuer: letsencrypt-dev spec: diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index 3b86dc55a..1d1eea758 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -30,6 +30,12 @@ resource "azurerm_kubernetes_cluster" "main" { vnet_subnet_id = var.system_node_pool_subnet_id pod_subnet_id = var.pod_subnet_id zones = local.zones + + upgrade_settings { + max_surge = "10%" + drain_timeout_in_minutes = 0 + node_soak_duration_in_minutes = 0 + } } identity { diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf index d00f3ca0d..3a445b0ba 100644 --- a/scenarios/AksOpenAiTerraform/terraform/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/outputs.tf @@ -6,6 +6,10 @@ output "cluster_name" { value = module.aks.name } +output "workload_managed_identity_client_id" { + value = azurerm_user_assigned_identity.aks_workload.client_id +} + output "acr_name" { value = module.container_registry.name } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index e9809190f..010435262 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -11,9 +11,4 @@ variable "location" { variable "kubernetes_version" { type = string default = "1.30.7" -} - -variable "email" { - type = string - default = "ariaamini@microsoft.com" } \ No newline at end of file From d4692f2e5ceeac3eb16af1e345f7e5bcb9938560 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 25 Feb 2025 23:41:13 -0500 Subject: [PATCH 085/119] wip --- scenarios/AksOpenAiTerraform/deploy.sh | 14 +- .../AksOpenAiTerraform/quickstart-app.yml | 16 +- .../terraform/.terraform.lock.hcl | 28 ++- .../AksOpenAiTerraform/terraform/main.tf | 182 +++--------------- .../terraform/modules/aks/main.tf | 27 +-- .../terraform/modules/aks/outputs.tf | 12 +- .../terraform/modules/bastion_host/main.tf | 55 ------ .../modules/bastion_host/variables.tf | 19 -- .../modules/container_registry/main.tf | 1 + .../terraform/modules/virtual_network/main.tf | 82 ++++++-- .../modules/virtual_network/variables.tf | 15 -- .../AksOpenAiTerraform/terraform/outputs.tf | 4 +- .../AksOpenAiTerraform/terraform/variables.tf | 10 + 13 files changed, 140 insertions(+), 325 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf diff --git a/scenarios/AksOpenAiTerraform/deploy.sh b/scenarios/AksOpenAiTerraform/deploy.sh index 44239d32a..69bc2f483 100644 --- a/scenarios/AksOpenAiTerraform/deploy.sh +++ b/scenarios/AksOpenAiTerraform/deploy.sh @@ -3,16 +3,15 @@ cd terraform export RESOURCE_GROUP=$(terraform output -raw resource_group_name) export CLUSTER_NAME=$(terraform output -raw cluster_name) -export WORKLOAD_MANAGED_IDENTITY_CLIENT_ID=$(terraform output -raw workload_managed_identity_client_id) -export ACR_NAME=$(terraform output -raw acr_name) +export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform output -raw workload_identity_client_id) cd .. + +export ACR_URL="privatelink.azurecr.io/magic8ball:v1" export SUBSCRIPTION_ID=$(az account show --query id --output tsv) export EMAIL="amini5454@gmail.com" # Build Image az acr login --name $ACR_NAME -export ACR_URL=$(az acr show --name $ACR_NAME --query loginServer --output tsv) -export IMAGE=$ACR_URL/magic8ball:v1 docker build -t $IMAGE ./app --push # Login @@ -25,8 +24,6 @@ helm repo add prometheus-community https://prometheus-community.github.io/helm-c helm repo update # NGINX ingress controller helm install ingress-nginx ingress-nginx/ingress-nginx \ - --create-namespace \ - --namespace "ingress-basic" \ --set controller.replicaCount=2 \ --set controller.nodeSelector."kubernetes\.io/os"=linux \ --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ @@ -36,18 +33,13 @@ helm install ingress-nginx ingress-nginx/ingress-nginx \ --set controller.metrics.serviceMonitor.additionalLabels.release="prometheus" # Cert manager helm install cert-manager jetstack/cert-manager \ - --create-namespace \ - --namespace cert-manager \ --set crds.enabled=true \ --set nodeSelector."kubernetes\.io/os"=linux # Prometheus helm install prometheus prometheus-community/kube-prometheus-stack \ - --create-namespace \ - --namespace prometheus \ --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false -kubectl create namespace magic8ball envsubst < quickstart-app.yml | kubectl apply -f - # Add DNS Record diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index 19d67ba2a..5ee8b6738 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ConfigMap metadata: name: magic8ball-configmap - namespace: magic8ball data: TITLE: "Magic 8 Ball" LABEL: "Pose your question and cross your fingers!" @@ -10,14 +9,13 @@ data: IMAGE_WIDTH: "80" AZURE_OPENAI_TYPE: azure_ad AZURE_OPENAI_BASE: https://myopenai.openai.azure.com/ - AZURE_OPENAI_MODEL: gpt-35-turbo - AZURE_OPENAI_DEPLOYMENT: magic8ballGPT + AZURE_OPENAI_MODEL: gpt-4o-mini + AZURE_OPENAI_DEPLOYMENT: gpt-4o-mini --- apiVersion: apps/v1 kind: Deployment metadata: name: magic8ball - namespace: magic8ball labels: app.kubernetes.io/name: magic8ball azure.workload.identity/use: "true" @@ -31,9 +29,11 @@ spec: labels: app.kubernetes.io/name: magic8ball spec: + serviceAccountName: magic8ball-sa containers: - name: magic8ball image: $IMAGE + imagePullPolicy: Always ports: - containerPort: 8501 envFrom: @@ -44,7 +44,6 @@ apiVersion: v1 kind: Service metadata: name: magic8ball-service - namespace: magic8ball spec: selector: app.kubernetes.io/name: magic8ball @@ -57,16 +56,14 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: magic8ball-service-account - namespace: magic8ball + name: magic8ball-sa annotations: - azure.workload.identity/client-id: $WORKLOAD_MANAGED_IDENTITY_CLIENT_ID + azure.workload.identity/client-id: $WORKLOAD_IDENTITY_CLIENT_ID --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: magic8ball-ingress - namespace: ingress-nginx annotations: cert-manager.io/cluster-issuer: letsencrypt-dev spec: @@ -91,7 +88,6 @@ apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: name: letsencrypt-dev - namespace: cert-manager spec: acme: server: https://acme-v02.api.letsencrypt.org/directory diff --git a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl index 8faadd9c3..6222f4e7e 100644 --- a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl +++ b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl @@ -6,7 +6,6 @@ provider "registry.terraform.io/hashicorp/azurerm" { constraints = "~> 4.16.0" hashes = [ "h1:7e25Wr4cpUvlAcwL+9ZOeeA1xha84LqTZNviDaVQFlo=", - "h1:UNZga7kYMfYfDHmuP6LvHmJNXlb3fyvRY1tA9ol6yY4=", "zh:2035e461a94bd4180557a06f8e56f228a8a035608d0dac4d08e5870cf9265276", "zh:3f15778a22ef1b9d0fa28670e5ea6ef1094b0be2533f43f350a2ef15d471b353", "zh:4f1a4d03b008dd958bcd6bf82cf088fbaa9c121be2fd35e10e6b06c6e8f6aaa1", @@ -23,21 +22,20 @@ provider "registry.terraform.io/hashicorp/azurerm" { } provider "registry.terraform.io/hashicorp/random" { - version = "3.6.3" + version = "3.7.1" hashes = [ - "h1:Fnaec9vA8sZ8BXVlN3Xn9Jz3zghSETIKg7ch8oXhxno=", - "h1:N2IQabOiZC5eCEGrfgVS6ChVmRDh1ENtfHgGjnV4QQQ=", - "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", - "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8", - "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe", - "zh:4fa45c44c0de582c2edb8a2e054f55124520c16a39b2dfc0355929063b6395b1", - "zh:588508280501a06259e023b0695f6a18149a3816d259655c424d068982cbdd36", - "zh:737c4d99a87d2a4d1ac0a54a73d2cb62974ccb2edbd234f333abd079a32ebc9e", + "h1:/qtweZW2sk0kBNiQM02RvBXmlVdI9oYqRMCyBZ8XA98=", + "zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585", + "zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef", + "zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d", + "zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6", + "zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1", + "zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e", + "zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:a357ab512e5ebc6d1fda1382503109766e21bbfdfaa9ccda43d313c122069b30", - "zh:c51bfb15e7d52cc1a2eaec2a903ac2aff15d162c172b1b4c17675190e8147615", - "zh:e0951ee6fa9df90433728b96381fb867e3db98f66f735e0c3e24f8f16903f0ad", - "zh:e3cdcb4e73740621dabd82ee6a37d6cfce7fee2a03d8074df65086760f5cf556", - "zh:eff58323099f1bd9a0bec7cb04f717e7f1b2774c7d612bf7581797e1622613a0", + "zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601", + "zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436", + "zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d", + "zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea", ] } diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index e2298c97c..51520da19 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -1,15 +1,7 @@ data "azurerm_client_config" "current" { } -resource "random_string" "rg_suffix" { - length = 6 - special = false - lower = false - upper = false - numeric = true -} - -resource "random_string" "storage_account_suffix" { +resource "random_string" "this" { length = 8 special = false lower = true @@ -20,15 +12,9 @@ resource "random_string" "storage_account_suffix" { locals { tenant_id = data.azurerm_client_config.current.tenant_id subscription_id = data.azurerm_client_config.current.subscription_id - random_id = random_string.rg_suffix.result - - namespace = "magic8ball" - service_account_name = "magic8ball-sa" + random_id = random_string.this.result } -############################################################################### -# Resource Group -############################################################################### resource "azurerm_resource_group" "main" { name = "${var.resource_group_name_prefix}-${local.random_id}-rg" location = var.location @@ -38,9 +24,6 @@ resource "azurerm_resource_group" "main" { } } -############################################################################### -# Application -############################################################################### module "openai" { source = "./modules/openai" name = "OpenAi-${local.random_id}" @@ -50,10 +33,10 @@ module "openai" { sku_name = "S0" deployments = [ { - name = "gpt-4" + name = var.model_name model = { - name = "gpt-4" - version = "turbo-2024-04-09" + name = var.model_name + version = var.model_version } } ] @@ -97,14 +80,14 @@ module "container_registry" { module "storage_account" { source = "./modules/storage_account" - name = "boot${random_string.storage_account_suffix.result}" + name = "boot${local.random_id}" location = var.location resource_group_name = azurerm_resource_group.main.name } module "key_vault" { source = "./modules/key_vault" - name = "KeyVault-${local.random_id}" + name = "KeyVault${local.random_id}" location = var.location resource_group_name = azurerm_resource_group.main.name @@ -124,9 +107,6 @@ module "log_analytics_workspace" { retention_in_days = 30 } -############################################################################### -# Networking -############################################################################### module "virtual_network" { source = "./modules/virtual_network" name = "AksVNet" @@ -134,143 +114,35 @@ module "virtual_network" { resource_group_name = azurerm_resource_group.main.name address_space = ["10.0.0.0/8"] - subnets = [ - { - name : "VmSubnet" - address_prefixes : ["10.243.1.0/24"] - }, - { - name : "AzureBastionSubnet" - address_prefixes : ["10.243.2.0/24"] - }, - { - name : "SystemSubnet" - address_prefixes : ["10.240.0.0/16"] - }, - { - name : "UserSubnet" - address_prefixes : ["10.241.0.0/16"] - }, - { - name : "PodSubnet" - address_prefixes : ["10.242.0.0/16"] - delegation = { - name = "delegation" - service_delegation = { - name = "Microsoft.ContainerService/managedClusters" - actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] - } - } - }, - ] - log_analytics_workspace_id = module.log_analytics_workspace.id } -module "nat_gateway" { - source = "./modules/nat_gateway" - name = "NatGateway" - location = var.location - resource_group_name = azurerm_resource_group.main.name - - subnet_ids = module.virtual_network.subnet_ids -} - -module "bastion_host" { - source = "./modules/bastion_host" - name = "BastionHost" - location = var.location - resource_group_name = azurerm_resource_group.main.name - - subnet_id = module.virtual_network.subnet_ids["AzureBastionSubnet"] - - log_analytics_workspace_id = module.log_analytics_workspace.id -} - -############################################################################### -# Private DNS Zones -############################################################################### -module "acr_private_dns_zone" { - source = "./modules/dns" - location = var.location - resource_group_name = azurerm_resource_group.main.name - - name = "privatelink.azurecr.io" - subresource_name = "account" - private_connection_resource_id = module.openai.id - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids["VmSubnet"] -} - -module "openai_private_dns_zone" { - source = "./modules/dns" - location = var.location - resource_group_name = azurerm_resource_group.main.name - - name = "privatelink.openai.azure.com" - subresource_name = "registry" - private_connection_resource_id = module.container_registry.id - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids["VmSubnet"] -} - -module "key_vault_private_dns_zone" { - source = "./modules/dns" - location = var.location - resource_group_name = azurerm_resource_group.main.name - - name = "privatelink.vaultcore.azure.net" - subresource_name = "vault" - private_connection_resource_id = module.key_vault.id - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids["VmSubnet"] -} - -module "blob_private_dns_zone" { - source = "./modules/dns" - location = var.location - resource_group_name = azurerm_resource_group.main.name - - name = "privatelink.blob.core.windows.net" - subresource_name = "blob" - private_connection_resource_id = module.storage_account.id - virtual_network_id = module.virtual_network.id - subnet_id = module.virtual_network.subnet_ids["VmSubnet"] -} - -############################################################################### -# Identities/Roles -############################################################################### -resource "azurerm_user_assigned_identity" "aks_workload" { - name = "WorkloadManagedIdentity" - resource_group_name = azurerm_resource_group.main.name - location = var.location -} - resource "azurerm_federated_identity_credential" "this" { - name = "${title(local.namespace)}FederatedIdentity" + name = "FederatedIdentity" resource_group_name = azurerm_resource_group.main.name audience = ["api://AzureADTokenExchange"] issuer = module.aks.oidc_issuer_url - parent_id = azurerm_user_assigned_identity.aks_workload.id - subject = "system:serviceaccount:${local.namespace}:${local.service_account_name}" + parent_id = module.aks.workload_identity.id + subject = "system:serviceaccount:default:magic8ball-sa" } -resource "azurerm_role_assignment" "cognitive_services_user_assignment" { - role_definition_name = "Cognitive Services User" - scope = module.openai.id - principal_id = azurerm_user_assigned_identity.aks_workload.principal_id -} +# resource "azurerm_role_assignment" "cognitive_services_user_assignment" { +# role_definition_name = "Cognitive Services User" +# scope = module.openai.id +# principal_id = module.aks.workload_identity_client_id +# } -resource "azurerm_role_assignment" "network_contributor_assignment" { - role_definition_name = "Network Contributor" - scope = azurerm_resource_group.main.id - principal_id = module.aks.aks_identity_principal_id -} +# resource "azurerm_role_assignment" "network_contributor_assignment" { +# role_definition_name = "Network Contributor" +# scope = azurerm_resource_group.main.id +# principal_id = module.aks.workload_identity_client_id +# } -resource "azurerm_role_assignment" "acr_pull_assignment" { - role_definition_name = "AcrPull" - scope = module.container_registry.id - principal_id = module.aks.kubelet_identity_object_id -} +# resource "azurerm_role_assignment" "acr_pull_assignment" { +# role_definition_name = "AcrPull" +# scope = module.container_registry.id +# principal_id = module.aks.workload_identity_client_id + +# skip_service_principal_aad_check = true +# } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf index 1d1eea758..dcd272782 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf @@ -1,9 +1,5 @@ -locals { - zones = ["2", "3"] -} - -resource "azurerm_user_assigned_identity" "aks" { - name = "${var.name}Identity" +resource "azurerm_user_assigned_identity" "workload" { + name = "WorkloadManagedIdentity" resource_group_name = var.resource_group_name location = var.location } @@ -13,7 +9,6 @@ resource "azurerm_kubernetes_cluster" "main" { location = var.location resource_group_name = var.resource_group_name kubernetes_version = var.kubernetes_version - dns_prefix = lower(var.name) automatic_upgrade_channel = "stable" sku_tier = var.sku_tier @@ -25,11 +20,8 @@ resource "azurerm_kubernetes_cluster" "main" { default_node_pool { name = "system" - node_count = 1 + node_count = 2 vm_size = var.system_node_pool_vm_size - vnet_subnet_id = var.system_node_pool_subnet_id - pod_subnet_id = var.pod_subnet_id - zones = local.zones upgrade_settings { max_surge = "10%" @@ -40,14 +32,12 @@ resource "azurerm_kubernetes_cluster" "main" { identity { type = "UserAssigned" - identity_ids = tolist([azurerm_user_assigned_identity.aks.id]) + identity_ids = tolist([azurerm_user_assigned_identity.workload.id]) } network_profile { - dns_service_ip = "10.2.0.10" - network_plugin = "azure" + network_plugin = "kubenet" outbound_type = "userAssignedNATGateway" - service_cidr = "10.2.0.0/24" } oms_agent { @@ -56,15 +46,12 @@ resource "azurerm_kubernetes_cluster" "main" { } } -resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { +resource "azurerm_kubernetes_cluster_node_pool" "this" { kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id name = "user" - vm_size = var.user_node_pool_vm_size mode = "User" - zones = local.zones - vnet_subnet_id = var.user_node_pool_subnet_id - pod_subnet_id = var.pod_subnet_id orchestrator_version = var.kubernetes_version + vm_size = var.user_node_pool_vm_size os_type = "Linux" priority = "Regular" } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf index 158f62992..8e9de73e9 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf @@ -6,12 +6,16 @@ output "id" { value = azurerm_kubernetes_cluster.main.id } -output "aks_identity_principal_id" { - value = azurerm_user_assigned_identity.aks.principal_id +output "workload_identity" { + value = azurerm_user_assigned_identity.workload } -output "kubelet_identity_object_id" { - value = azurerm_kubernetes_cluster.main.kubelet_identity.0.object_id +output "workload_identity_client_id" { + value = azurerm_user_assigned_identity.workload.client_id +} + +output "kubelet_identity" { + value = azurerm_kubernetes_cluster.main.kubelet_identity.0 } output "oidc_issuer_url" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf deleted file mode 100644 index 4066b7c17..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/main.tf +++ /dev/null @@ -1,55 +0,0 @@ -resource "azurerm_public_ip" "public_ip" { - name = "${var.name}PublicIp" - location = var.location - resource_group_name = var.resource_group_name - allocation_method = "Static" - sku = "Standard" -} - -resource "azurerm_bastion_host" "bastion_host" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - - ip_configuration { - name = "configuration" - subnet_id = var.subnet_id - public_ip_address_id = azurerm_public_ip.public_ip.id - } -} - -resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "BastionDiagnosticsSettings" - target_resource_id = azurerm_bastion_host.bastion_host.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - enabled_log { - category = "BastionAuditLogs" - } - - metric { - category = "AllMetrics" - } -} - -resource "azurerm_monitor_diagnostic_setting" "pip_settings" { - name = "BastionDdosDiagnosticsSettings" - target_resource_id = azurerm_public_ip.public_ip.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - enabled_log { - category = "DDoSProtectionNotifications" - } - - enabled_log { - category = "DDoSMitigationFlowLogs" - } - - enabled_log { - category = "DDoSMitigationReports" - } - - metric { - category = "AllMetrics" - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf deleted file mode 100644 index c3b2d0b5d..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/bastion_host/variables.tf +++ /dev/null @@ -1,19 +0,0 @@ -variable "name" { - type = string -} - -variable "location" { - type = string -} - -variable "resource_group_name" { - type = string -} - -variable "subnet_id" { - type = string -} - -variable "log_analytics_workspace_id" { - type = string -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf index d071ad376..ef978ab0a 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf @@ -3,6 +3,7 @@ resource "azurerm_container_registry" "acr" { resource_group_name = var.resource_group_name location = var.location sku = var.sku + anonymous_pull_enabled = true } resource "azurerm_monitor_diagnostic_setting" "settings" { diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf index 30f5fe5cd..e37bf8401 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf @@ -1,29 +1,73 @@ resource "azurerm_virtual_network" "vnet" { name = var.name + location = var.location + resource_group_name = var.resource_group_name + address_space = var.address_space +} + +resource "azurerm_subnet" "bastion" { + name = "AzureBastionSubnet" + resource_group_name = var.resource_group_name + + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = ["10.243.2.0/24"] +} + +resource "azurerm_public_ip" "public_ip" { + name = "PublicIp" location = var.location resource_group_name = var.resource_group_name + + allocation_method = "Static" + sku = "Standard" } -resource "azurerm_subnet" "subnet" { - for_each = { for subnet in var.subnets : subnet.name => subnet } - - name = each.key - resource_group_name = var.resource_group_name - virtual_network_name = azurerm_virtual_network.vnet.name - address_prefixes = each.value.address_prefixes - private_endpoint_network_policies = "Enabled" - - dynamic "delegation" { - for_each = each.value.delegation != null ? [each.value.delegation] : [] - content { - name = "delegation" - - service_delegation { - name = delegation.value.service_delegation.name - actions = delegation.value.service_delegation.actions - } - } +resource "azurerm_bastion_host" "bastion_host" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + + ip_configuration { + name = "configuration" + subnet_id = azurerm_subnet.bastion.id + public_ip_address_id = azurerm_public_ip.public_ip.id + } +} + +resource "azurerm_monitor_diagnostic_setting" "settings" { + name = "BastionDiagnosticsSettings" + target_resource_id = azurerm_bastion_host.bastion_host.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "BastionAuditLogs" + } + + metric { + category = "AllMetrics" + } +} + +resource "azurerm_monitor_diagnostic_setting" "pip_settings" { + name = "BastionDdosDiagnosticsSettings" + target_resource_id = azurerm_public_ip.public_ip.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "DDoSProtectionNotifications" + } + + enabled_log { + category = "DDoSMitigationFlowLogs" + } + + enabled_log { + category = "DDoSMitigationReports" + } + + metric { + category = "AllMetrics" } } diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf index fcadabecf..a10213502 100644 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf @@ -14,21 +14,6 @@ variable "address_space" { type = list(string) } -variable "subnets" { - description = "Subnets configuration" - type = list(object({ - name = string - address_prefixes = list(string) - delegation = optional(object({ - name = string, - service_delegation = object({ - name = string - actions = list(string) - }) - })) - })) -} - variable "log_analytics_workspace_id" { type = string } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf index 3a445b0ba..42d0a25af 100644 --- a/scenarios/AksOpenAiTerraform/terraform/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/outputs.tf @@ -6,8 +6,8 @@ output "cluster_name" { value = module.aks.name } -output "workload_managed_identity_client_id" { - value = azurerm_user_assigned_identity.aks_workload.client_id +output "workload_identity_client_id" { + value = module.aks.workload_identity.client_id } output "acr_name" { diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf index 010435262..69dfab821 100644 --- a/scenarios/AksOpenAiTerraform/terraform/variables.tf +++ b/scenarios/AksOpenAiTerraform/terraform/variables.tf @@ -11,4 +11,14 @@ variable "location" { variable "kubernetes_version" { type = string default = "1.30.7" +} + +variable "model_name" { + type = string + default = "gpt-4o-mini" +} + +variable "model_version" { + type = string + default = "2024-07-18" } \ No newline at end of file From 502d08b82a58d5d426971a694b286c81f9a4df1f Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 00:52:30 -0500 Subject: [PATCH 086/119] Fix --- scenarios/AksOpenAiTerraform/deploy.sh | 6 +- .../terraform/.terraform.lock.hcl | 41 --- .../AksOpenAiTerraform/terraform/main.tf | 237 ++++++++++++------ .../terraform/modules/aks/main.tf | 95 ------- .../terraform/modules/aks/outputs.tf | 23 -- .../terraform/modules/aks/variables.tf | 51 ---- .../modules/container_registry/main.tf | 25 -- .../modules/container_registry/outputs.tf | 7 - .../modules/container_registry/variables.tf | 19 -- .../terraform/modules/dns/main.tf | 25 -- .../terraform/modules/dns/variables.tf | 27 -- .../terraform/modules/key_vault/main.tf | 37 --- .../terraform/modules/key_vault/outputs.tf | 7 - .../terraform/modules/key_vault/variables.tf | 23 -- .../terraform/modules/log_analytics/main.tf | 20 -- .../terraform/modules/log_analytics/output.tf | 3 - .../modules/log_analytics/variables.tf | 19 -- .../terraform/modules/nat_gateway/main.tf | 23 -- .../modules/nat_gateway/variables.tf | 15 -- .../terraform/modules/openai/main.tf | 53 ---- .../terraform/modules/openai/output.tf | 34 --- .../terraform/modules/openai/variables.tf | 33 --- .../terraform/modules/storage_account/main.tf | 12 - .../modules/storage_account/outputs.tf | 3 - .../modules/storage_account/variables.tf | 11 - .../terraform/modules/virtual_network/main.tf | 82 ------ .../modules/virtual_network/outputs.tf | 11 - .../modules/virtual_network/variables.tf | 19 -- .../AksOpenAiTerraform/terraform/outputs.tf | 10 +- 29 files changed, 163 insertions(+), 808 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf delete mode 100644 scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf diff --git a/scenarios/AksOpenAiTerraform/deploy.sh b/scenarios/AksOpenAiTerraform/deploy.sh index 69bc2f483..41de3536d 100644 --- a/scenarios/AksOpenAiTerraform/deploy.sh +++ b/scenarios/AksOpenAiTerraform/deploy.sh @@ -6,8 +6,8 @@ export CLUSTER_NAME=$(terraform output -raw cluster_name) export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform output -raw workload_identity_client_id) cd .. -export ACR_URL="privatelink.azurecr.io/magic8ball:v1" export SUBSCRIPTION_ID=$(az account show --query id --output tsv) +export ACR_URL="privatelink.azurecr.io/magic8ball:v1" export EMAIL="amini5454@gmail.com" # Build Image @@ -22,7 +22,6 @@ helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx helm repo add jetstack https://charts.jetstack.io helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update -# NGINX ingress controller helm install ingress-nginx ingress-nginx/ingress-nginx \ --set controller.replicaCount=2 \ --set controller.nodeSelector."kubernetes\.io/os"=linux \ @@ -31,15 +30,14 @@ helm install ingress-nginx ingress-nginx/ingress-nginx \ --set controller.metrics.enabled=true \ --set controller.metrics.serviceMonitor.enabled=true \ --set controller.metrics.serviceMonitor.additionalLabels.release="prometheus" -# Cert manager helm install cert-manager jetstack/cert-manager \ --set crds.enabled=true \ --set nodeSelector."kubernetes\.io/os"=linux -# Prometheus helm install prometheus prometheus-community/kube-prometheus-stack \ --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false +# Run deployment envsubst < quickstart-app.yml | kubectl apply -f - # Add DNS Record diff --git a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl deleted file mode 100644 index 6222f4e7e..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl +++ /dev/null @@ -1,41 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/azurerm" { - version = "4.16.0" - constraints = "~> 4.16.0" - hashes = [ - "h1:7e25Wr4cpUvlAcwL+9ZOeeA1xha84LqTZNviDaVQFlo=", - "zh:2035e461a94bd4180557a06f8e56f228a8a035608d0dac4d08e5870cf9265276", - "zh:3f15778a22ef1b9d0fa28670e5ea6ef1094b0be2533f43f350a2ef15d471b353", - "zh:4f1a4d03b008dd958bcd6bf82cf088fbaa9c121be2fd35e10e6b06c6e8f6aaa1", - "zh:5859f31c342364e849b4f8c437a46f33e927fa820244d0732b8d2ec74a95712d", - "zh:693d0f15512ca8c6b5e999b3a7551503feb06b408b3836bc6a6403e518b9ddab", - "zh:7f4912bec5b04f5156935292377c12484c13582151eb3c2555df409a7e5fb6e0", - "zh:bb9a509497f3a131c52fac32348919bf1b9e06c69a65f24607b03f7b56fb47b6", - "zh:c1b0c64e49ac591fd038ad71e71403ff71c07476e27e8da718c29f0028ea6d0d", - "zh:dd4ca432ee14eb0bb0cdc0bb463c8675b8ef02497be870a20d8dfee3e7fe52b3", - "zh:df58bb7fea984d2b11709567842ca4d55b3f24e187aa6be99e3677f55cbbe7da", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f7fb37704da50c096f9c7c25e8a95fe73ce1d3c5aab0d616d506f07bc5cfcdd8", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.7.1" - hashes = [ - "h1:/qtweZW2sk0kBNiQM02RvBXmlVdI9oYqRMCyBZ8XA98=", - "zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585", - "zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef", - "zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d", - "zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6", - "zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1", - "zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e", - "zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601", - "zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436", - "zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d", - "zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea", - ] -} diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 51520da19..527c02782 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -18,131 +18,210 @@ locals { resource "azurerm_resource_group" "main" { name = "${var.resource_group_name_prefix}-${local.random_id}-rg" location = var.location +} - lifecycle { - ignore_changes = [tags] - } +############################################################################### +# Kubernetes +############################################################################### +resource "azurerm_user_assigned_identity" "workload" { + name = "WorkloadManagedIdentity" + resource_group_name = azurerm_resource_group.main.name + location = var.location } -module "openai" { - source = "./modules/openai" - name = "OpenAi-${local.random_id}" +resource "azurerm_federated_identity_credential" "this" { + name = "FederatedIdentity" + resource_group_name = azurerm_resource_group.main.name + + audience = ["api://AzureADTokenExchange"] + issuer = azurerm_kubernetes_cluster.main.oidc_issuer_url + parent_id = azurerm_user_assigned_identity.workload.id + subject = "system:serviceaccount:default:magic8ball-sa" +} + +resource "azurerm_kubernetes_cluster" "main" { + name = "AksCluster" location = var.location resource_group_name = azurerm_resource_group.main.name - sku_name = "S0" - deployments = [ - { - name = var.model_name - model = { - name = var.model_name - version = var.model_version - } + dns_prefix = "AksCluster${local.random_id}" + kubernetes_version = var.kubernetes_version + automatic_upgrade_channel = "stable" + sku_tier = "Standard" + + image_cleaner_enabled = true + image_cleaner_interval_hours = 72 + + workload_identity_enabled = true + oidc_issuer_enabled = true + + default_node_pool { + name = "system" + node_count = 2 + vm_size = "Standard_DS2_v2" + + upgrade_settings { + max_surge = "10%" + drain_timeout_in_minutes = 0 + node_soak_duration_in_minutes = 0 } - ] - custom_subdomain_name = "magic8ball-${local.random_id}" + } - log_analytics_workspace_id = module.log_analytics_workspace.id + identity { + type = "UserAssigned" + identity_ids = tolist([azurerm_user_assigned_identity.workload.id]) + } + + network_profile { + network_plugin = "kubenet" + outbound_type = "userAssignedNATGateway" + } } -module "aks" { - source = "./modules/aks" - name = "AksCluster" +resource "azurerm_kubernetes_cluster_node_pool" "this" { + kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id + name = "user" + mode = "User" + orchestrator_version = var.kubernetes_version + vm_size = "Standard_DS2_v2" + os_type = "Linux" + priority = "Regular" +} + +############################################################################### +# OpenAI +############################################################################### +resource "azurerm_cognitive_account" "openai" { + name = "OpenAi-${local.random_id}" location = var.location resource_group_name = azurerm_resource_group.main.name - resource_group_id = azurerm_resource_group.main.id - tenant_id = local.tenant_id - kubernetes_version = var.kubernetes_version - sku_tier = "Standard" - system_node_pool_vm_size = "Standard_DS2_v2" - user_node_pool_vm_size = "Standard_DS2_v2" + kind = "OpenAI" + custom_subdomain_name = "magic8ball-${local.random_id}" + sku_name = "S0" + public_network_access_enabled = true - system_node_pool_subnet_id = module.virtual_network.subnet_ids["SystemSubnet"] - user_node_pool_subnet_id = module.virtual_network.subnet_ids["UserSubnet"] - pod_subnet_id = module.virtual_network.subnet_ids["PodSubnet"] + identity { + type = "SystemAssigned" + } +} - log_analytics_workspace_id = module.log_analytics_workspace.id +resource "azurerm_cognitive_deployment" "deployment" { + name = var.model_name + cognitive_account_id = azurerm_cognitive_account.openai.id - depends_on = [module.nat_gateway] + model { + format = "OpenAI" + name = var.model_name + version = var.model_version + } + + sku { + name = "Standard" + } } -module "container_registry" { - source = "./modules/container_registry" - name = "acr${local.random_id}" +############################################################################### +# Key Vault +############################################################################### +resource "azurerm_key_vault" "this" { + name = "KeyVault${local.random_id}" location = var.location resource_group_name = azurerm_resource_group.main.name + tenant_id = local.tenant_id - sku = "Premium" + sku_name = "standard" + enabled_for_deployment = true + enabled_for_disk_encryption = true + enabled_for_template_deployment = true + enable_rbac_authorization = true + purge_protection_enabled = false + soft_delete_retention_days = 30 + + network_acls { + bypass = "AzureServices" + default_action = "Allow" + } +} - log_analytics_workspace_id = module.log_analytics_workspace.id +############################################################################### +# Container Registry +############################################################################### +resource "azurerm_container_registry" "this" { + name = "acr${local.random_id}" + resource_group_name = azurerm_resource_group.main.name + location = var.location + sku = "Premium" + anonymous_pull_enabled = true } -module "storage_account" { - source = "./modules/storage_account" +############################################################################### +# Storage Account +############################################################################### +resource "azurerm_storage_account" "storage_account" { name = "boot${local.random_id}" location = var.location resource_group_name = azurerm_resource_group.main.name + + account_kind = "StorageV2" + account_tier = "Standard" + account_replication_type = "LRS" + is_hns_enabled = false + + allow_nested_items_to_be_public = false } -module "key_vault" { - source = "./modules/key_vault" - name = "KeyVault${local.random_id}" +############################################################################### +# Networking +############################################################################### +resource "azurerm_virtual_network" "this" { + name = "Vnet" location = var.location resource_group_name = azurerm_resource_group.main.name + address_space = ["10.0.0.0/8"] +} - tenant_id = local.tenant_id - sku_name = "standard" +resource "azurerm_subnet" "this" { + name = "AzureBastionSubnet" + resource_group_name = azurerm_resource_group.main.name - log_analytics_workspace_id = module.log_analytics_workspace.id + virtual_network_name = azurerm_virtual_network.this.name + address_prefixes = ["10.243.2.0/24"] } -module "log_analytics_workspace" { - source = "./modules/log_analytics" - name = "Workspace" +resource "azurerm_public_ip" "this" { + name = "PublicIp" location = var.location resource_group_name = azurerm_resource_group.main.name - sku = "PerGB2018" - retention_in_days = 30 + allocation_method = "Static" + sku = "Standard" } -module "virtual_network" { - source = "./modules/virtual_network" - name = "AksVNet" +resource "azurerm_bastion_host" "this" { + name = "BastionHost" location = var.location resource_group_name = azurerm_resource_group.main.name - address_space = ["10.0.0.0/8"] - log_analytics_workspace_id = module.log_analytics_workspace.id + ip_configuration { + name = "configuration" + subnet_id = azurerm_subnet.this.id + public_ip_address_id = azurerm_public_ip.this.id + } } -resource "azurerm_federated_identity_credential" "this" { - name = "FederatedIdentity" +resource "azurerm_nat_gateway" "this" { + name = "NatGateway" + location = var.location resource_group_name = azurerm_resource_group.main.name - - audience = ["api://AzureADTokenExchange"] - issuer = module.aks.oidc_issuer_url - parent_id = module.aks.workload_identity.id - subject = "system:serviceaccount:default:magic8ball-sa" } -# resource "azurerm_role_assignment" "cognitive_services_user_assignment" { -# role_definition_name = "Cognitive Services User" -# scope = module.openai.id -# principal_id = module.aks.workload_identity_client_id -# } - -# resource "azurerm_role_assignment" "network_contributor_assignment" { -# role_definition_name = "Network Contributor" -# scope = azurerm_resource_group.main.id -# principal_id = module.aks.workload_identity_client_id -# } - -# resource "azurerm_role_assignment" "acr_pull_assignment" { -# role_definition_name = "AcrPull" -# scope = module.container_registry.id -# principal_id = module.aks.workload_identity_client_id +resource "azurerm_subnet_nat_gateway_association" "gateway_association" { + subnet_id = azurerm_subnet.this.id + nat_gateway_id = azurerm_nat_gateway.this.id +} -# skip_service_principal_aad_check = true -# } \ No newline at end of file +resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_association" { + nat_gateway_id = azurerm_nat_gateway.this.id + public_ip_address_id = azurerm_public_ip.this.id +} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf deleted file mode 100644 index dcd272782..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/main.tf +++ /dev/null @@ -1,95 +0,0 @@ -resource "azurerm_user_assigned_identity" "workload" { - name = "WorkloadManagedIdentity" - resource_group_name = var.resource_group_name - location = var.location -} - -resource "azurerm_kubernetes_cluster" "main" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - kubernetes_version = var.kubernetes_version - automatic_upgrade_channel = "stable" - sku_tier = var.sku_tier - - image_cleaner_enabled = true - image_cleaner_interval_hours = 72 - - workload_identity_enabled = true - oidc_issuer_enabled = true - - default_node_pool { - name = "system" - node_count = 2 - vm_size = var.system_node_pool_vm_size - - upgrade_settings { - max_surge = "10%" - drain_timeout_in_minutes = 0 - node_soak_duration_in_minutes = 0 - } - } - - identity { - type = "UserAssigned" - identity_ids = tolist([azurerm_user_assigned_identity.workload.id]) - } - - network_profile { - network_plugin = "kubenet" - outbound_type = "userAssignedNATGateway" - } - - oms_agent { - msi_auth_for_monitoring_enabled = true - log_analytics_workspace_id = var.log_analytics_workspace_id - } -} - -resource "azurerm_kubernetes_cluster_node_pool" "this" { - kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id - name = "user" - mode = "User" - orchestrator_version = var.kubernetes_version - vm_size = var.user_node_pool_vm_size - os_type = "Linux" - priority = "Regular" -} - -resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "AksDiagnosticsSettings" - target_resource_id = azurerm_kubernetes_cluster.main.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - enabled_log { - category = "kube-apiserver" - } - - enabled_log { - category = "kube-audit" - } - - enabled_log { - category = "kube-audit-admin" - } - - enabled_log { - category = "kube-controller-manager" - } - - enabled_log { - category = "kube-scheduler" - } - - enabled_log { - category = "cluster-autoscaler" - } - - enabled_log { - category = "guard" - } - - metric { - category = "AllMetrics" - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf deleted file mode 100644 index 8e9de73e9..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -output "name" { - value = azurerm_kubernetes_cluster.main.name -} - -output "id" { - value = azurerm_kubernetes_cluster.main.id -} - -output "workload_identity" { - value = azurerm_user_assigned_identity.workload -} - -output "workload_identity_client_id" { - value = azurerm_user_assigned_identity.workload.client_id -} - -output "kubelet_identity" { - value = azurerm_kubernetes_cluster.main.kubelet_identity.0 -} - -output "oidc_issuer_url" { - value = azurerm_kubernetes_cluster.main.oidc_issuer_url -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf deleted file mode 100644 index c0e76833b..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/aks/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -variable "name" { - type = string -} - -variable "resource_group_name" { - type = string -} - -variable "resource_group_id" { - type = string -} - -variable "location" { - type = string -} - -variable "tenant_id" { - type = string -} - -variable "kubernetes_version" { - type = string -} - -variable "sku_tier" { - type = string -} - -variable "system_node_pool_vm_size" { - type = string -} - -variable "user_node_pool_vm_size" { - type = string -} - -variable "log_analytics_workspace_id" { - type = string -} - -variable "user_node_pool_subnet_id" { - type = string -} - -variable "system_node_pool_subnet_id" { - type = string -} - -variable "pod_subnet_id" { - type = string -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf deleted file mode 100644 index ef978ab0a..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/main.tf +++ /dev/null @@ -1,25 +0,0 @@ -resource "azurerm_container_registry" "acr" { - name = var.name - resource_group_name = var.resource_group_name - location = var.location - sku = var.sku - anonymous_pull_enabled = true -} - -resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "ContainerDiagnosticsSettings" - target_resource_id = azurerm_container_registry.acr.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - enabled_log { - category = "ContainerRegistryRepositoryEvents" - } - - enabled_log { - category = "ContainerRegistryLoginEvents" - } - - metric { - category = "AllMetrics" - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf deleted file mode 100644 index 9642edb0a..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/outputs.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "name" { - value = azurerm_container_registry.acr.name -} - -output "id" { - value = azurerm_container_registry.acr.id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf deleted file mode 100644 index df252b035..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/container_registry/variables.tf +++ /dev/null @@ -1,19 +0,0 @@ -variable "name" { - type = string -} - -variable "resource_group_name" { - type = string -} - -variable "location" { - type = string -} - -variable "sku" { - type = string -} - -variable "log_analytics_workspace_id" { - type = string -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf deleted file mode 100644 index bf65750a4..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/dns/main.tf +++ /dev/null @@ -1,25 +0,0 @@ -resource "azurerm_private_dns_zone" "this" { - name = var.name - resource_group_name = var.resource_group_name -} - -resource "azurerm_private_dns_zone_virtual_network_link" "this" { - name = "link_to_${lower(basename(azurerm_private_dns_zone.this.name))}" - resource_group_name = var.resource_group_name - private_dns_zone_name = azurerm_private_dns_zone.this.name - virtual_network_id = var.virtual_network_id -} - -resource "azurerm_private_endpoint" "this" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - subnet_id = var.subnet_id - - private_service_connection { - name = "connection" - private_connection_resource_id = var.private_connection_resource_id - is_manual_connection = false - subresource_names = [var.subresource_name] - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf deleted file mode 100644 index 8933b684a..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/dns/variables.tf +++ /dev/null @@ -1,27 +0,0 @@ -variable "name" { - type = string -} - -variable "location" { - type = string -} - -variable "resource_group_name" { - type = string -} - -variable "subresource_name" { - type = string -} - -variable "virtual_network_id" { - type = string -} - -variable "subnet_id" { - type = string -} - -variable "private_connection_resource_id" { - type = string -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf deleted file mode 100644 index a23b4448f..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -resource "azurerm_key_vault" "key_vault" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - tenant_id = var.tenant_id - - sku_name = var.sku_name - enabled_for_deployment = true - enabled_for_disk_encryption = true - enabled_for_template_deployment = true - enable_rbac_authorization = true - purge_protection_enabled = false - soft_delete_retention_days = 30 - - network_acls { - bypass = "AzureServices" - default_action = "Allow" - } -} - -resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "KeyVaultDiagnosticsSettings" - target_resource_id = azurerm_key_vault.key_vault.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - enabled_log { - category = "AuditEvent" - } - - enabled_log { - category = "AzurePolicyEvaluationDetails" - } - - metric { - category = "AllMetrics" - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf deleted file mode 100644 index ffb395cc4..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/outputs.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "name" { - value = azurerm_key_vault.key_vault.name -} - -output "id" { - value = azurerm_key_vault.key_vault.id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf deleted file mode 100644 index 2918ab083..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/key_vault/variables.tf +++ /dev/null @@ -1,23 +0,0 @@ -variable "name" { - type = string -} - -variable "location" { - type = string -} - -variable "resource_group_name" { - type = string -} - -variable "tenant_id" { - type = string -} - -variable "sku_name" { - type = string -} - -variable "log_analytics_workspace_id" { - type = string -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf deleted file mode 100644 index e3c50d5d5..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/main.tf +++ /dev/null @@ -1,20 +0,0 @@ -resource "azurerm_log_analytics_workspace" "this" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - sku = var.sku - retention_in_days = var.retention_in_days -} - -resource "azurerm_log_analytics_solution" "this" { - solution_name = "ContainerInsights" - location = var.location - resource_group_name = var.resource_group_name - workspace_resource_id = azurerm_log_analytics_workspace.this.id - workspace_name = azurerm_log_analytics_workspace.this.name - - plan { - product = "OMSGallery/ContainerInsights" - publisher = "Microsoft" - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf deleted file mode 100644 index 837cd9e49..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/output.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "id" { - value = azurerm_log_analytics_workspace.this.id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf deleted file mode 100644 index 9c1aa1f04..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/log_analytics/variables.tf +++ /dev/null @@ -1,19 +0,0 @@ -variable "resource_group_name" { - type = string -} - -variable "location" { - type = string -} - -variable "name" { - type = string -} - -variable "sku" { - type = string -} - -variable "retention_in_days" { - type = number -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf deleted file mode 100644 index 1cb1cae21..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/main.tf +++ /dev/null @@ -1,23 +0,0 @@ -resource "azurerm_nat_gateway" "this" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name -} - -resource "azurerm_public_ip" "nat_gateway" { - name = "${var.name}PublicIp" - location = var.location - resource_group_name = var.resource_group_name - allocation_method = "Static" -} - -resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_association" { - nat_gateway_id = azurerm_nat_gateway.this.id - public_ip_address_id = azurerm_public_ip.nat_gateway.id -} - -resource "azurerm_subnet_nat_gateway_association" "gateway_association" { - for_each = var.subnet_ids - subnet_id = each.value - nat_gateway_id = azurerm_nat_gateway.this.id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf deleted file mode 100644 index 0accf9ced..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/nat_gateway/variables.tf +++ /dev/null @@ -1,15 +0,0 @@ -variable "name" { - type = string -} - -variable "location" { - type = string -} - -variable "resource_group_name" { - type = string -} - -variable "subnet_ids" { - type = map(string) -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf deleted file mode 100644 index 8821e5ce6..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/main.tf +++ /dev/null @@ -1,53 +0,0 @@ -resource "azurerm_cognitive_account" "openai" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - - kind = "OpenAI" - custom_subdomain_name = var.custom_subdomain_name - sku_name = var.sku_name - public_network_access_enabled = true - - identity { - type = "SystemAssigned" - } -} - -resource "azurerm_cognitive_deployment" "deployment" { - for_each = { for deployment in var.deployments : deployment.name => deployment } - - name = each.key - cognitive_account_id = azurerm_cognitive_account.openai.id - - model { - format = "OpenAI" - name = each.value.model.name - version = each.value.model.version - } - - sku { - name = "Standard" - } -} - -resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "OpenAiDiagnosticsSettings" - target_resource_id = azurerm_cognitive_account.openai.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - enabled_log { - category = "Audit" - } - - enabled_log { - category = "RequestResponse" - } - - enabled_log { - category = "Trace" - } - - metric { - category = "AllMetrics" - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf deleted file mode 100644 index 2b3e7cb0c..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/output.tf +++ /dev/null @@ -1,34 +0,0 @@ -output "id" { - value = azurerm_cognitive_account.openai.id - description = "Specifies the resource id of the log analytics workspace" -} - -output "location" { - value = azurerm_cognitive_account.openai.location - description = "Specifies the location of the log analytics workspace" -} - -output "name" { - value = azurerm_cognitive_account.openai.name - description = "Specifies the name of the log analytics workspace" -} - -output "resource_group_name" { - value = azurerm_cognitive_account.openai.resource_group_name - description = "Specifies the name of the resource group that contains the log analytics workspace" -} - -output "endpoint" { - value = azurerm_cognitive_account.openai.endpoint - description = "Specifies the endpoint of the Azure OpenAI Service." -} - -output "primary_access_key" { - value = azurerm_cognitive_account.openai.endpoint - description = "Specifies the primary access key of the Azure OpenAI Service." -} - -output "secondary_access_key" { - value = azurerm_cognitive_account.openai.endpoint - description = "Specifies the secondary access key of the Azure OpenAI Service." -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf deleted file mode 100644 index 2eee76ed2..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/openai/variables.tf +++ /dev/null @@ -1,33 +0,0 @@ -variable "name" { - type = string -} - -variable "location" { - type = string -} - -variable "resource_group_name" { - type = string -} - -variable "sku_name" { - type = string -} - -variable "custom_subdomain_name" { - type = string -} - -variable "deployments" { - type = list(object({ - name = string - model = object({ - name = string - version = string - }) - })) -} - -variable "log_analytics_workspace_id" { - type = string -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf deleted file mode 100644 index 7d265fa25..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/main.tf +++ /dev/null @@ -1,12 +0,0 @@ -resource "azurerm_storage_account" "storage_account" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - - account_kind = "StorageV2" - account_tier = "Standard" - account_replication_type = "LRS" - is_hns_enabled = false - - allow_nested_items_to_be_public = false -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf deleted file mode 100644 index 156c1d8d7..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "id" { - value = azurerm_storage_account.storage_account.id -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf deleted file mode 100644 index 3d2c4d24d..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/storage_account/variables.tf +++ /dev/null @@ -1,11 +0,0 @@ -variable "name" { - type = string -} - -variable "location" { - type = string -} - -variable "resource_group_name" { - type = string -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf deleted file mode 100644 index e37bf8401..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/main.tf +++ /dev/null @@ -1,82 +0,0 @@ -resource "azurerm_virtual_network" "vnet" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - - address_space = var.address_space -} - -resource "azurerm_subnet" "bastion" { - name = "AzureBastionSubnet" - resource_group_name = var.resource_group_name - - virtual_network_name = azurerm_virtual_network.vnet.name - address_prefixes = ["10.243.2.0/24"] -} - -resource "azurerm_public_ip" "public_ip" { - name = "PublicIp" - location = var.location - resource_group_name = var.resource_group_name - - allocation_method = "Static" - sku = "Standard" -} - -resource "azurerm_bastion_host" "bastion_host" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - - ip_configuration { - name = "configuration" - subnet_id = azurerm_subnet.bastion.id - public_ip_address_id = azurerm_public_ip.public_ip.id - } -} - -resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "BastionDiagnosticsSettings" - target_resource_id = azurerm_bastion_host.bastion_host.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - enabled_log { - category = "BastionAuditLogs" - } - - metric { - category = "AllMetrics" - } -} - -resource "azurerm_monitor_diagnostic_setting" "pip_settings" { - name = "BastionDdosDiagnosticsSettings" - target_resource_id = azurerm_public_ip.public_ip.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - enabled_log { - category = "DDoSProtectionNotifications" - } - - enabled_log { - category = "DDoSMitigationFlowLogs" - } - - enabled_log { - category = "DDoSMitigationReports" - } - - metric { - category = "AllMetrics" - } -} - -resource "azurerm_monitor_diagnostic_setting" "settings" { - name = "VirtualNetworkDiagnosticsSettings" - target_resource_id = azurerm_virtual_network.vnet.id - log_analytics_workspace_id = var.log_analytics_workspace_id - - metric { - category = "AllMetrics" - } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf deleted file mode 100644 index 32c5d99f0..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/outputs.tf +++ /dev/null @@ -1,11 +0,0 @@ -output "name" { - value = azurerm_virtual_network.vnet.name -} - -output "id" { - value = azurerm_virtual_network.vnet.id -} - -output "subnet_ids" { - value = { for subnet in azurerm_subnet.subnet : subnet.name => subnet.id } -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf b/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf deleted file mode 100644 index a10213502..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/modules/virtual_network/variables.tf +++ /dev/null @@ -1,19 +0,0 @@ -variable "name" { - type = string -} - -variable "location" { - type = string -} - -variable "resource_group_name" { - type = string -} - -variable "address_space" { - type = list(string) -} - -variable "log_analytics_workspace_id" { - type = string -} \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf index 42d0a25af..706c664c4 100644 --- a/scenarios/AksOpenAiTerraform/terraform/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/outputs.tf @@ -1,15 +1,11 @@ output "resource_group_name" { - value = azurerm_resource_group.main.name + value = azurerm_resource_group.main.name } output "cluster_name" { - value = module.aks.name + value = azurerm_kubernetes_cluster.main.name } output "workload_identity_client_id" { - value = module.aks.workload_identity.client_id -} - -output "acr_name" { - value = module.container_registry.name + value = azurerm_user_assigned_identity.workload.client_id } \ No newline at end of file From 91933dae6ce3f16e46f67ad90a479d210426aedd Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 02:54:21 -0500 Subject: [PATCH 087/119] WIP --- scenarios/AksOpenAiTerraform/README.md | 14 +- scenarios/AksOpenAiTerraform/deploy.sh | 4 +- .../{app => magic8ball}/Dockerfile | 0 .../{app => magic8ball}/app.py | 0 .../{app => magic8ball}/icons/magic8ball.png | Bin .../{app => magic8ball}/icons/robot.png | Bin .../{app => magic8ball}/requirements.txt | 0 .../terraform/.terraform.lock.hcl | 41 +++++ .../AksOpenAiTerraform/terraform/main.tf | 153 ++++++++---------- .../AksOpenAiTerraform/terraform/outputs.tf | 4 + 10 files changed, 117 insertions(+), 99 deletions(-) rename scenarios/AksOpenAiTerraform/{app => magic8ball}/Dockerfile (100%) rename scenarios/AksOpenAiTerraform/{app => magic8ball}/app.py (100%) rename scenarios/AksOpenAiTerraform/{app => magic8ball}/icons/magic8ball.png (100%) rename scenarios/AksOpenAiTerraform/{app => magic8ball}/icons/robot.png (100%) rename scenarios/AksOpenAiTerraform/{app => magic8ball}/requirements.txt (100%) create mode 100644 scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index 8497390a3..225432902 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -17,22 +17,10 @@ az extension add --name aks-preview az aks install-cli ``` -## Set up Subscription ID to authenticate for Terraform - -Terraform uses the ARM_SUBSCRIPTION_ID environment variable to authenticate while using CLI. +## Run Terraform ```bash export ARM_SUBSCRIPTION_ID=$SUBSCRIPTION_ID -``` - -## Init Terraform - -```bash terraform init -``` - -## Run Terraform - -```bash terraform apply ``` diff --git a/scenarios/AksOpenAiTerraform/deploy.sh b/scenarios/AksOpenAiTerraform/deploy.sh index 41de3536d..8f61e0626 100644 --- a/scenarios/AksOpenAiTerraform/deploy.sh +++ b/scenarios/AksOpenAiTerraform/deploy.sh @@ -4,11 +4,13 @@ cd terraform export RESOURCE_GROUP=$(terraform output -raw resource_group_name) export CLUSTER_NAME=$(terraform output -raw cluster_name) export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform output -raw workload_identity_client_id) +export ACR_NAME=$(terraform output -raw acr_name) cd .. +# Delete export SUBSCRIPTION_ID=$(az account show --query id --output tsv) -export ACR_URL="privatelink.azurecr.io/magic8ball:v1" export EMAIL="amini5454@gmail.com" +export IMAGE="acrguqrbpys.azurecr.io/magic8ball:v1" # Build Image az acr login --name $ACR_NAME diff --git a/scenarios/AksOpenAiTerraform/app/Dockerfile b/scenarios/AksOpenAiTerraform/magic8ball/Dockerfile similarity index 100% rename from scenarios/AksOpenAiTerraform/app/Dockerfile rename to scenarios/AksOpenAiTerraform/magic8ball/Dockerfile diff --git a/scenarios/AksOpenAiTerraform/app/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py similarity index 100% rename from scenarios/AksOpenAiTerraform/app/app.py rename to scenarios/AksOpenAiTerraform/magic8ball/app.py diff --git a/scenarios/AksOpenAiTerraform/app/icons/magic8ball.png b/scenarios/AksOpenAiTerraform/magic8ball/icons/magic8ball.png similarity index 100% rename from scenarios/AksOpenAiTerraform/app/icons/magic8ball.png rename to scenarios/AksOpenAiTerraform/magic8ball/icons/magic8ball.png diff --git a/scenarios/AksOpenAiTerraform/app/icons/robot.png b/scenarios/AksOpenAiTerraform/magic8ball/icons/robot.png similarity index 100% rename from scenarios/AksOpenAiTerraform/app/icons/robot.png rename to scenarios/AksOpenAiTerraform/magic8ball/icons/robot.png diff --git a/scenarios/AksOpenAiTerraform/app/requirements.txt b/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt similarity index 100% rename from scenarios/AksOpenAiTerraform/app/requirements.txt rename to scenarios/AksOpenAiTerraform/magic8ball/requirements.txt diff --git a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl new file mode 100644 index 000000000..6222f4e7e --- /dev/null +++ b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl @@ -0,0 +1,41 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.16.0" + constraints = "~> 4.16.0" + hashes = [ + "h1:7e25Wr4cpUvlAcwL+9ZOeeA1xha84LqTZNviDaVQFlo=", + "zh:2035e461a94bd4180557a06f8e56f228a8a035608d0dac4d08e5870cf9265276", + "zh:3f15778a22ef1b9d0fa28670e5ea6ef1094b0be2533f43f350a2ef15d471b353", + "zh:4f1a4d03b008dd958bcd6bf82cf088fbaa9c121be2fd35e10e6b06c6e8f6aaa1", + "zh:5859f31c342364e849b4f8c437a46f33e927fa820244d0732b8d2ec74a95712d", + "zh:693d0f15512ca8c6b5e999b3a7551503feb06b408b3836bc6a6403e518b9ddab", + "zh:7f4912bec5b04f5156935292377c12484c13582151eb3c2555df409a7e5fb6e0", + "zh:bb9a509497f3a131c52fac32348919bf1b9e06c69a65f24607b03f7b56fb47b6", + "zh:c1b0c64e49ac591fd038ad71e71403ff71c07476e27e8da718c29f0028ea6d0d", + "zh:dd4ca432ee14eb0bb0cdc0bb463c8675b8ef02497be870a20d8dfee3e7fe52b3", + "zh:df58bb7fea984d2b11709567842ca4d55b3f24e187aa6be99e3677f55cbbe7da", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f7fb37704da50c096f9c7c25e8a95fe73ce1d3c5aab0d616d506f07bc5cfcdd8", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.7.1" + hashes = [ + "h1:/qtweZW2sk0kBNiQM02RvBXmlVdI9oYqRMCyBZ8XA98=", + "zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585", + "zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef", + "zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d", + "zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6", + "zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1", + "zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e", + "zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601", + "zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436", + "zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d", + "zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea", + ] +} diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 527c02782..4a9d39708 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -18,47 +18,34 @@ locals { resource "azurerm_resource_group" "main" { name = "${var.resource_group_name_prefix}-${local.random_id}-rg" location = var.location + + lifecycle { + ignore_changes = [tags] + } } ############################################################################### # Kubernetes ############################################################################### -resource "azurerm_user_assigned_identity" "workload" { - name = "WorkloadManagedIdentity" - resource_group_name = azurerm_resource_group.main.name - location = var.location -} - -resource "azurerm_federated_identity_credential" "this" { - name = "FederatedIdentity" - resource_group_name = azurerm_resource_group.main.name - - audience = ["api://AzureADTokenExchange"] - issuer = azurerm_kubernetes_cluster.main.oidc_issuer_url - parent_id = azurerm_user_assigned_identity.workload.id - subject = "system:serviceaccount:default:magic8ball-sa" -} - resource "azurerm_kubernetes_cluster" "main" { name = "AksCluster" location = var.location resource_group_name = azurerm_resource_group.main.name - dns_prefix = "AksCluster${local.random_id}" + sku_tier = "Standard" kubernetes_version = var.kubernetes_version + dns_prefix = "AksCluster${local.random_id}" automatic_upgrade_channel = "stable" - sku_tier = "Standard" + workload_identity_enabled = true + oidc_issuer_enabled = true image_cleaner_enabled = true image_cleaner_interval_hours = 72 - workload_identity_enabled = true - oidc_issuer_enabled = true - default_node_pool { - name = "system" - node_count = 2 + name = "agentpool" vm_size = "Standard_DS2_v2" + node_count = 2 upgrade_settings { max_surge = "10%" @@ -71,23 +58,36 @@ resource "azurerm_kubernetes_cluster" "main" { type = "UserAssigned" identity_ids = tolist([azurerm_user_assigned_identity.workload.id]) } - - network_profile { - network_plugin = "kubenet" - outbound_type = "userAssignedNATGateway" - } } resource "azurerm_kubernetes_cluster_node_pool" "this" { + name = "userpool" + mode = "User" + node_count = 2 + kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id - name = "user" - mode = "User" orchestrator_version = var.kubernetes_version vm_size = "Standard_DS2_v2" os_type = "Linux" priority = "Regular" } +resource "azurerm_user_assigned_identity" "workload" { + name = "WorkloadManagedIdentity" + resource_group_name = azurerm_resource_group.main.name + location = var.location +} + +resource "azurerm_federated_identity_credential" "this" { + name = "FederatedIdentity" + resource_group_name = azurerm_resource_group.main.name + + audience = ["api://AzureADTokenExchange"] + issuer = azurerm_kubernetes_cluster.main.oidc_issuer_url + parent_id = azurerm_user_assigned_identity.workload.id + subject = "system:serviceaccount:default:magic8ball-sa" +} + ############################################################################### # OpenAI ############################################################################### @@ -121,6 +121,44 @@ resource "azurerm_cognitive_deployment" "deployment" { } } +############################################################################### +# Networking +############################################################################### +resource "azurerm_virtual_network" "this" { + name = "Vnet" + location = var.location + resource_group_name = azurerm_resource_group.main.name + + address_space = ["10.0.0.0/8"] +} + +resource "azurerm_subnet" "this" { + name = "AzureBastionSubnet" + resource_group_name = azurerm_resource_group.main.name + + virtual_network_name = azurerm_virtual_network.this.name + address_prefixes = ["10.243.2.0/24"] +} + +resource "azurerm_public_ip" "this" { + name = "PublicIp" + location = var.location + resource_group_name = azurerm_resource_group.main.name + allocation_method = "Static" +} + +resource "azurerm_bastion_host" "this" { + name = "BastionHost" + location = var.location + resource_group_name = azurerm_resource_group.main.name + + ip_configuration { + name = "configuration" + subnet_id = azurerm_subnet.this.id + public_ip_address_id = azurerm_public_ip.this.id + } +} + ############################################################################### # Key Vault ############################################################################### @@ -169,59 +207,4 @@ resource "azurerm_storage_account" "storage_account" { is_hns_enabled = false allow_nested_items_to_be_public = false -} - -############################################################################### -# Networking -############################################################################### -resource "azurerm_virtual_network" "this" { - name = "Vnet" - location = var.location - resource_group_name = azurerm_resource_group.main.name - address_space = ["10.0.0.0/8"] -} - -resource "azurerm_subnet" "this" { - name = "AzureBastionSubnet" - resource_group_name = azurerm_resource_group.main.name - - virtual_network_name = azurerm_virtual_network.this.name - address_prefixes = ["10.243.2.0/24"] -} - -resource "azurerm_public_ip" "this" { - name = "PublicIp" - location = var.location - resource_group_name = azurerm_resource_group.main.name - - allocation_method = "Static" - sku = "Standard" -} - -resource "azurerm_bastion_host" "this" { - name = "BastionHost" - location = var.location - resource_group_name = azurerm_resource_group.main.name - - ip_configuration { - name = "configuration" - subnet_id = azurerm_subnet.this.id - public_ip_address_id = azurerm_public_ip.this.id - } -} - -resource "azurerm_nat_gateway" "this" { - name = "NatGateway" - location = var.location - resource_group_name = azurerm_resource_group.main.name -} - -resource "azurerm_subnet_nat_gateway_association" "gateway_association" { - subnet_id = azurerm_subnet.this.id - nat_gateway_id = azurerm_nat_gateway.this.id -} - -resource "azurerm_nat_gateway_public_ip_association" "nat_gategay_public_ip_association" { - nat_gateway_id = azurerm_nat_gateway.this.id - public_ip_address_id = azurerm_public_ip.this.id } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf index 706c664c4..aba79ba15 100644 --- a/scenarios/AksOpenAiTerraform/terraform/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/outputs.tf @@ -8,4 +8,8 @@ output "cluster_name" { output "workload_identity_client_id" { value = azurerm_user_assigned_identity.workload.client_id +} + +output "acr_name" { + value = azurerm_container_registry.this.name } \ No newline at end of file From 6d77e53185a781f063f3b60049098b4798a66779 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 02:54:35 -0500 Subject: [PATCH 088/119] rename --- .../AksOpenAiTerraform/{terraform => infra}/.terraform.lock.hcl | 0 scenarios/AksOpenAiTerraform/{terraform => infra}/main.tf | 0 scenarios/AksOpenAiTerraform/{terraform => infra}/outputs.tf | 0 scenarios/AksOpenAiTerraform/{terraform => infra}/provider.tf | 0 scenarios/AksOpenAiTerraform/{terraform => infra}/variables.tf | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename scenarios/AksOpenAiTerraform/{terraform => infra}/.terraform.lock.hcl (100%) rename scenarios/AksOpenAiTerraform/{terraform => infra}/main.tf (100%) rename scenarios/AksOpenAiTerraform/{terraform => infra}/outputs.tf (100%) rename scenarios/AksOpenAiTerraform/{terraform => infra}/provider.tf (100%) rename scenarios/AksOpenAiTerraform/{terraform => infra}/variables.tf (100%) diff --git a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/infra/.terraform.lock.hcl similarity index 100% rename from scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl rename to scenarios/AksOpenAiTerraform/infra/.terraform.lock.hcl diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/infra/main.tf similarity index 100% rename from scenarios/AksOpenAiTerraform/terraform/main.tf rename to scenarios/AksOpenAiTerraform/infra/main.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/infra/outputs.tf similarity index 100% rename from scenarios/AksOpenAiTerraform/terraform/outputs.tf rename to scenarios/AksOpenAiTerraform/infra/outputs.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/provider.tf b/scenarios/AksOpenAiTerraform/infra/provider.tf similarity index 100% rename from scenarios/AksOpenAiTerraform/terraform/provider.tf rename to scenarios/AksOpenAiTerraform/infra/provider.tf diff --git a/scenarios/AksOpenAiTerraform/terraform/variables.tf b/scenarios/AksOpenAiTerraform/infra/variables.tf similarity index 100% rename from scenarios/AksOpenAiTerraform/terraform/variables.tf rename to scenarios/AksOpenAiTerraform/infra/variables.tf From 0d689d5b65389cb844e426c18a7cc0f7012fec8b Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 05:02:16 -0500 Subject: [PATCH 089/119] Working --- scenarios/AksOpenAiTerraform/README.md | 82 ++++++++++++++++++- scenarios/AksOpenAiTerraform/deploy.sh | 51 ------------ .../AksOpenAiTerraform/quickstart-app.yml | 26 +++--- 3 files changed, 91 insertions(+), 68 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/deploy.sh diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index 225432902..5d9415531 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -17,10 +17,86 @@ az extension add --name aks-preview az aks install-cli ``` -## Run Terraform +## Provision Resources + +Provision all infrastructure using terraform. ```bash +export SUBSCRIPTION_ID="b7684763-6bf2-4be5-8fdd-f9fadb0f27a1" +export EMAIL="amini5454@gmail.com" + export ARM_SUBSCRIPTION_ID=$SUBSCRIPTION_ID -terraform init -terraform apply +terraform -chdir=infra init +terraform -chdir=infra apply + +# Save outputs +export RESOURCE_GROUP=$(terraform -chdir=infra output -raw resource_group_name) +export CLUSTER_NAME=$(terraform -chdir=infra output -raw cluster_name) +export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform -chdir=infra output -raw workload_identity_client_id) +export ACR_NAME=$(terraform -chdir=infra output -raw acr_name) +``` + +# Login + +Login to AKS cluster + +```bash +az aks get-credentials --admin --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --subscription $SUBSCRIPTION_ID +``` + +## Build Dockerfile + +Build app's container image + +```bash +export IMAGE="$ACR_NAME.azurecr.io/magic8ball:v1" +az acr login --name $ACR_NAME +docker build -t $IMAGE ./magic8ball --push +``` + +# Deploy App + +```bash +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo add jetstack https://charts.jetstack.io +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update + +helm install ingress-nginx ingress-nginx/ingress-nginx \ + --set controller.replicaCount=2 \ + --set controller.nodeSelector."kubernetes\.io/os"=linux \ + --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz \ + --set controller.metrics.enabled=true \ + --set controller.metrics.serviceMonitor.enabled=true \ + --set controller.metrics.serviceMonitor.additionalLabels.release="prometheus" +helm install cert-manager jetstack/cert-manager \ + --set crds.enabled=true \ + --set nodeSelector."kubernetes\.io/os"=linux +helm install prometheus prometheus-community/kube-prometheus-stack \ + --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ + --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false + +envsubst < quickstart-app.yml | kubectl apply -f - +``` + +# Wait for App to Finish + +Wait for public IP + +```bash +kubectl wait --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' ingress/magic8ball-ingress ``` + +# Add DNS Record + +Have DNS point to app + +```bash +PUBLIC_IP=$(kubectl get ingress magic8ball-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +az network dns record-set a add-record \ + --zone-name "contoso.com" \ + --resource-group $RESOURCE_GROUP \ + --record-set-name magic8ball \ + --ipv4-address $PUBLIC_IP +``` \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/deploy.sh b/scenarios/AksOpenAiTerraform/deploy.sh deleted file mode 100644 index 8f61e0626..000000000 --- a/scenarios/AksOpenAiTerraform/deploy.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -cd terraform -export RESOURCE_GROUP=$(terraform output -raw resource_group_name) -export CLUSTER_NAME=$(terraform output -raw cluster_name) -export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform output -raw workload_identity_client_id) -export ACR_NAME=$(terraform output -raw acr_name) -cd .. - -# Delete -export SUBSCRIPTION_ID=$(az account show --query id --output tsv) -export EMAIL="amini5454@gmail.com" -export IMAGE="acrguqrbpys.azurecr.io/magic8ball:v1" - -# Build Image -az acr login --name $ACR_NAME -docker build -t $IMAGE ./app --push - -# Login -az aks get-credentials --admin --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --subscription $SUBSCRIPTION_ID - -# Install Deps -helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx -helm repo add jetstack https://charts.jetstack.io -helm repo add prometheus-community https://prometheus-community.github.io/helm-charts -helm repo update -helm install ingress-nginx ingress-nginx/ingress-nginx \ - --set controller.replicaCount=2 \ - --set controller.nodeSelector."kubernetes\.io/os"=linux \ - --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ - --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz \ - --set controller.metrics.enabled=true \ - --set controller.metrics.serviceMonitor.enabled=true \ - --set controller.metrics.serviceMonitor.additionalLabels.release="prometheus" -helm install cert-manager jetstack/cert-manager \ - --set crds.enabled=true \ - --set nodeSelector."kubernetes\.io/os"=linux -helm install prometheus prometheus-community/kube-prometheus-stack \ - --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ - --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false - -# Run deployment -envsubst < quickstart-app.yml | kubectl apply -f - - -# Add DNS Record -PUBLIC_IP=$(kubectl get ingress magic8ball-ingress -n magic8ball -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -az network dns record-set a add-record \ - --zone-name "contoso.com" \ - --resource-group $RESOURCE_GROUP \ - --record-set-name magic8ball \ - --ipv4-address $PUBLIC_IP \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index 5ee8b6738..4564df825 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -33,7 +33,6 @@ spec: containers: - name: magic8ball image: $IMAGE - imagePullPolicy: Always ports: - containerPort: 8501 envFrom: @@ -43,23 +42,16 @@ spec: apiVersion: v1 kind: Service metadata: - name: magic8ball-service + name: magic8ball spec: selector: app.kubernetes.io/name: magic8ball type: ClusterIP ports: - protocol: TCP - port: 8501 + port: 80 targetPort: 8501 --- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: magic8ball-sa - annotations: - azure.workload.identity/client-id: $WORKLOAD_IDENTITY_CLIENT_ID ---- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -73,8 +65,7 @@ spec: - magic8ball.contoso.com secretName: tls-secret rules: - - host: magic8ball.contoso.com - http: + - http: paths: - path: / pathType: Prefix @@ -82,7 +73,7 @@ spec: service: name: magic8ball port: - number: 8501 + number: 80 --- apiVersion: cert-manager.io/v1 kind: ClusterIssuer @@ -97,4 +88,11 @@ spec: solvers: - http01: ingress: - ingressClassName: nginx \ No newline at end of file + ingressClassName: nginx +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: magic8ball-sa + annotations: + azure.workload.identity/client-id: $WORKLOAD_IDENTITY_CLIENT_ID \ No newline at end of file From 175eda522b9bfb32e8c8eb1e204cb70a7d386af0 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 06:08:37 -0500 Subject: [PATCH 090/119] Add venv to gitignore --- scenarios/AksOpenAiTerraform/.gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/.gitignore b/scenarios/AksOpenAiTerraform/.gitignore index 21e6d3cbd..06a16355e 100644 --- a/scenarios/AksOpenAiTerraform/.gitignore +++ b/scenarios/AksOpenAiTerraform/.gitignore @@ -34,4 +34,6 @@ override.tf.json # Ignore CLI configuration files .terraformrc -terraform.rc \ No newline at end of file +terraform.rc + +.venv \ No newline at end of file From d677471f1b97339a21f4fe32db3fcfbfac84276e Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 06:10:20 -0500 Subject: [PATCH 091/119] del --- scenarios/AksOpenAiTerraform/magic8ball/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt b/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt index ec7c03c8b..5e9fdc7a7 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt +++ b/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt @@ -1,4 +1,3 @@ -python-dotenv==0.19.2 streamlit==1.22.0 streamlit-chat==0.0.2.2 azure-identity==1.13.0 From d34bd3a98686ff1c21cd8ed9d5a52cc5e1c88eaa Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 06:17:18 -0500 Subject: [PATCH 092/119] ui tweaks --- .../AksOpenAiTerraform/magic8ball/app.py | 106 ++++-------------- 1 file changed, 19 insertions(+), 87 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index ee012133c..6fa822a24 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -2,50 +2,25 @@ # # Make sure to provide a value for the following environment variables: # - AZURE_OPENAI_BASE (ex: https://eastus.api.cognitive.microsoft.com/) -# - AZURE_OPENAI_KEY # - AZURE_OPENAI_DEPLOYMENT # - AZURE_OPENAI_MODEL -# - TITLE -# - TEMPERATURE -# - SYSTEM (Used to describe the assistant's personality.) -# -# You can use two different authentication methods: -# -# - API key: set the AZURE_OPENAI_TYPE environment variable to azure and the AZURE_OPENAI_KEY environment variable to the key of -# your Azure OpenAI resource. You can use the regional endpoint, such as https://eastus.api.cognitive.microsoft.com/, passed in -# the AZURE_OPENAI_BASE environment variable, to connect to the Azure OpenAI resource. -# -# - Azure Active Directory: set the AZURE_OPENAI_TYPE environment variable to azure_ad and use a service principal or managed -# identity with the DefaultAzureCredential object to acquire a token. For more information on the DefaultAzureCredential in Python, -# see https://docs.microsoft.com/en-us/azure/developer/python/azure-sdk-authenticate?tabs=cmd -# Make sure to assign the "Cognitive Services User" role to the service principal or managed identity used to authenticate to -# Azure OpenAI. For more information, see https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/managed-identity. -# If you want to use Azure AD integrated security, you need to create a custom subdomain for your Azure OpenAI resource and use the -# specific endpoint containing the custom domain, such as https://bingo.openai.azure.com/ where bingo is the custom subdomain. -# If you specify the regional endpoint, you get a wonderful error: "Subdomain does not map to a resource.". -# Hence, make sure to pass the endpoint containing the custom domain in the AZURE_OPENAI_BASE environment variable. -# -# Use the following command to run the app: -# - streamlit run app.py +# - AZURE_OPENAI_VERSION import os -import sys import time import openai import logging import streamlit as st from streamlit_chat import message from azure.identity import DefaultAzureCredential -from dotenv import load_dotenv -from dotenv import dotenv_values - -# Load environment variables from .env file -if os.path.exists(".env"): - load_dotenv(override=True) - config = dotenv_values(".env") # Read environment variables -assistan_profile = """ +api_base = os.getenv("AZURE_OPENAI_BASE") +api_version = os.environ.get("AZURE_OPENAI_VERSION") +engine = os.getenv("AZURE_OPENAI_DEPLOYMENT") +model = os.getenv("AZURE_OPENAI_MODEL") + +system = """ You are the infamous Magic 8 Ball. You need to randomly reply to any question with one of the following answers: - It is certain. @@ -72,58 +47,21 @@ Add a short comment in a pirate style at the end! Follow your heart and be creative! For mor information, see https://en.wikipedia.org/wiki/Magic_8_Ball """ -title = os.environ.get("TITLE", "Magic 8 Ball") -text_input_label = os.environ.get("TEXT_INPUT_LABEL", "Pose your question and cross your fingers!") -image_file_name = os.environ.get("IMAGE_FILE_NAME", "magic8ball.png") -image_width = int(os.environ.get("IMAGE_WIDTH", 80)) -temperature = float(os.environ.get("TEMPERATURE", 0.9)) -system = os.environ.get("SYSTEM", assistan_profile) -api_base = os.getenv("AZURE_OPENAI_BASE") -api_key = os.getenv("AZURE_OPENAI_KEY") -api_type = os.environ.get("AZURE_OPENAI_TYPE", "azure") -api_version = os.environ.get("AZURE_OPENAI_VERSION", "2023-05-15") -engine = os.getenv("AZURE_OPENAI_DEPLOYMENT") -model = os.getenv("AZURE_OPENAI_MODEL") +title = "Magic 8 Ball" +text_input_label = "Pose your question and cross your fingers!" +image_file_name = "magic8ball.png" +image_width = 80 +temperature = 0.9 # Configure OpenAI -openai.api_type = api_type +openai.api_type = "azure" openai.api_version = api_version openai.api_base = api_base -# Set default Azure credential -default_credential = DefaultAzureCredential() if openai.api_type == "azure_ad" else None - -# Configure a logger -logging.basicConfig(stream = sys.stdout, - format = '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s', - level = logging.INFO) -logger = logging.getLogger(__name__) - -# Log variables -logger.info(f"title: {title}") -logger.info(f"text_input_label: {text_input_label}") -logger.info(f"image_file_name: {image_file_name}") -logger.info(f"image_width: {image_width}") -logger.info(f"temperature: {temperature}") -logger.info(f"system: {system}") -logger.info(f"api_base: {api_base}") -logger.info(f"api_key: {api_key}") -logger.info(f"api_type: {api_type}") -logger.info(f"api_version: {api_version}") -logger.info(f"engine: {engine}") -logger.info(f"model: {model}") - # Authenticate to Azure OpenAI -if openai.api_type == "azure": - openai.api_key = api_key -elif openai.api_type == "azure_ad": - openai_token = default_credential.get_token("https://cognitiveservices.azure.com/.default") - openai.api_key = openai_token.token - if 'openai_token' not in st.session_state: - st.session_state['openai_token'] = openai_token -else: - logger.error("Invalid API type. Please set the AZURE_OPENAI_TYPE environment variable to azure or azure_ad.") - raise ValueError("Invalid API type. Please set the AZURE_OPENAI_TYPE environment variable to azure or azure_ad.") +default_credential = DefaultAzureCredential() +openai_token = default_credential.get_token("https://cognitiveservices.azure.com/.default") +openai.api_key = openai_token.token # Customize Streamlit UI using CSS st.markdown(""" @@ -299,12 +237,6 @@ def user_change(): # - normal: display the chat history as a list of messages using the streamlit_chat message() function # - rich: display the chat history as a list of messages using the Streamlit markdown() function if st.session_state['generated']: - tab1, tab2 = st.tabs(["normal", "rich"]) - with tab1: - for i in range(len(st.session_state['generated']) - 1, -1, -1): - message(st.session_state['past'][i], is_user = True, key = str(i) + '_user', avatar_style = "fun-emoji", seed = "Nala") - message(st.session_state['generated'][i], key = str(i), avatar_style = "bottts", seed = "Fluffy") - with tab2: - for i in range(len(st.session_state['generated']) - 1, -1, -1): - st.markdown(st.session_state['past'][i]) - st.markdown(st.session_state['generated'][i]) \ No newline at end of file + for i in range(len(st.session_state['generated']) - 1, -1, -1): + message(st.session_state['past'][i], is_user = True, key = str(i) + '_user', avatar_style = "fun-emoji", seed = "Nala") + message(st.session_state['generated'][i], key = str(i), avatar_style = "bottts", seed = "Fluffy") \ No newline at end of file From 69921b3a7e3dac3bcc1d35e9109967305ec6b2cb Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 06:48:32 -0500 Subject: [PATCH 093/119] Fix --- .../AksOpenAiTerraform/magic8ball/app.py | 185 +++++++----------- 1 file changed, 74 insertions(+), 111 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index 6fa822a24..52cb0a04f 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -14,12 +14,16 @@ from streamlit_chat import message from azure.identity import DefaultAzureCredential -# Read environment variables -api_base = os.getenv("AZURE_OPENAI_BASE") +# Environment Variables api_version = os.environ.get("AZURE_OPENAI_VERSION") engine = os.getenv("AZURE_OPENAI_DEPLOYMENT") model = os.getenv("AZURE_OPENAI_MODEL") +title = "Magic 8 Ball" +text_input_label = "Pose your question and cross your fingers!" +image_file_name = "magic8ball.png" +image_width = 80 +temperature = 0.9 system = """ You are the infamous Magic 8 Ball. You need to randomly reply to any question with one of the following answers: @@ -47,21 +51,77 @@ Add a short comment in a pirate style at the end! Follow your heart and be creative! For mor information, see https://en.wikipedia.org/wiki/Magic_8_Ball """ -title = "Magic 8 Ball" -text_input_label = "Pose your question and cross your fingers!" -image_file_name = "magic8ball.png" -image_width = 80 -temperature = 0.9 - -# Configure OpenAI -openai.api_type = "azure" -openai.api_version = api_version -openai.api_base = api_base # Authenticate to Azure OpenAI default_credential = DefaultAzureCredential() openai_token = default_credential.get_token("https://cognitiveservices.azure.com/.default") -openai.api_key = openai_token.token + +# Init session_state +if 'prompts' not in st.session_state: + st.session_state['prompts'] = [{"role": "system", "content": system}] +if 'generated' not in st.session_state: + st.session_state['generated'] = [] +if 'past' not in st.session_state: + st.session_state['past'] = [] + + +def on_submit(): + # Avoid handling the event twice when clicking the Send button + chat_input = st.session_state['user'] + st.session_state['user'] = "" + if (chat_input == '' or + (len(st.session_state['past']) > 0 and chat_input == st.session_state['past'][-1])): + return + + # Save history + st.session_state['past'].append(chat_input) + + # Refresh token every 45 min + if st.session_state.get('openai_token') and st.session_state['openai_token'].expires_on < int(time.time()) - 45 * 60: + st.session_state['openai_token'] = default_credential.get_token("https://cognitiveservices.azure.com/.default") + openai.api_key = st.session_state['openai_token'].token + + # Generate API response + st.session_state['prompts'].append({"role": "user", "content": chat_input}) + completion = openai.ChatCompletion.create( + engine = engine, + model = model, + messages = st.session_state['prompts'], + temperature = temperature, + ) + message = completion.choices[0].message.content + st.session_state['generated'].append(message) + st.session_state['prompts'].append({"role": "assistant", "content": message}) + + +def reset(): + st.session_state['prompts'] = [{"role": "system", "content": system}] + st.session_state['past'] = [] + st.session_state['generated'] = [] + st.session_state['user'] = "" + + +# Row 1 +col1, col2 = st.columns([1, 7]) +with col1: # Robot icon + st.image(image = os.path.join("icons", image_file_name), width = image_width) +with col2: # Title + st.title(title) + +# Row 2 +col3, col4, col5 = st.columns([7, 1, 1]) +with col3: # Text box + user_input = st.text_input(text_input_label, key = "user", on_change = on_submit) +with col4: # 'Send' Button + st.button(label = "Send") +with col5: # 'New' Button + st.button(label = "New", on_click = reset) + +if st.session_state['generated']: + for i in range(len(st.session_state['generated']) - 1, -1, -1): + message(st.session_state['past'][i], is_user = True, key = str(i) + '_user', avatar_style = "fun-emoji", seed = "Nala") + message(st.session_state['generated'][i], key = str(i), avatar_style = "bottts", seed = "Fluffy") + # Customize Streamlit UI using CSS st.markdown(""" @@ -142,101 +202,4 @@ } } -""", unsafe_allow_html=True) - -# Initialize Streamlit session state -if 'prompts' not in st.session_state: - st.session_state['prompts'] = [{"role": "system", "content": system}] - -if 'generated' not in st.session_state: - st.session_state['generated'] = [] - -if 'past' not in st.session_state: - st.session_state['past'] = [] - -# Refresh the OpenAI security token every 45 minutes -def refresh_openai_token(): - if st.session_state['openai_token'].expires_on < int(time.time()) - 45 * 60: - st.session_state['openai_token'] = default_credential.get_token("https://cognitiveservices.azure.com/.default") - openai.api_key = st.session_state['openai_token'].token - -# Send user prompt to Azure OpenAI -def generate_response(prompt): - try: - st.session_state['prompts'].append({"role": "user", "content": prompt}) - - if openai.api_type == "azure_ad": - refresh_openai_token() - - completion = openai.ChatCompletion.create( - engine = engine, - model = model, - messages = st.session_state['prompts'], - temperature = temperature, - ) - - message = completion.choices[0].message.content - return message - except Exception as e: - logging.exception(f"Exception in generate_response: {e}") - -# Reset Streamlit session state to start a new chat from scratch -def new_click(): - st.session_state['prompts'] = [{"role": "system", "content": system}] - st.session_state['past'] = [] - st.session_state['generated'] = [] - st.session_state['user'] = "" - -# Handle on_change event for user input -def user_change(): - # Avoid handling the event twice when clicking the Send button - chat_input = st.session_state['user'] - st.session_state['user'] = "" - if (chat_input == '' or - (len(st.session_state['past']) > 0 and chat_input == st.session_state['past'][-1])): - return - - # Generate response invoking Azure OpenAI LLM - if chat_input != '': - output = generate_response(chat_input) - - # store the output - st.session_state['past'].append(chat_input) - st.session_state['generated'].append(output) - st.session_state['prompts'].append({"role": "assistant", "content": output}) - -# Create a 2-column layout. Note: Streamlit columns do not properly render on mobile devices. -# For more information, see https://github.com/streamlit/streamlit/issues/5003 -col1, col2 = st.columns([1, 7]) - -# Display the robot image -with col1: - st.image(image = os.path.join("icons", image_file_name), width = image_width) - -# Display the title -with col2: - st.title(title) - -# Create a 3-column layout. Note: Streamlit columns do not properly render on mobile devices. -# For more information, see https://github.com/streamlit/streamlit/issues/5003 -col3, col4, col5 = st.columns([7, 1, 1]) - -# Create text input in column 1 -with col3: - user_input = st.text_input(text_input_label, key = "user", on_change = user_change) - -# Create send button in column 2 -with col4: - st.button(label = "Send") - -# Create new button in column 3 -with col5: - st.button(label = "New", on_click = new_click) - -# Display the chat history in two separate tabs -# - normal: display the chat history as a list of messages using the streamlit_chat message() function -# - rich: display the chat history as a list of messages using the Streamlit markdown() function -if st.session_state['generated']: - for i in range(len(st.session_state['generated']) - 1, -1, -1): - message(st.session_state['past'][i], is_user = True, key = str(i) + '_user', avatar_style = "fun-emoji", seed = "Nala") - message(st.session_state['generated'][i], key = str(i), avatar_style = "bottts", seed = "Fluffy") \ No newline at end of file +""") \ No newline at end of file From 2ef60b7b9c7af28765621084e675e9634023a6c5 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 16:27:53 -0500 Subject: [PATCH 094/119] fix --- scenarios/AksOpenAiTerraform/.gitignore | 3 ++- scenarios/AksOpenAiTerraform/magic8ball/requirements.txt | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/.gitignore b/scenarios/AksOpenAiTerraform/.gitignore index 06a16355e..1b2971f39 100644 --- a/scenarios/AksOpenAiTerraform/.gitignore +++ b/scenarios/AksOpenAiTerraform/.gitignore @@ -36,4 +36,5 @@ override.tf.json .terraformrc terraform.rc -.venv \ No newline at end of file +.venv +.vscode \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt b/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt index 5e9fdc7a7..379071ce4 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt +++ b/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt @@ -1,4 +1,4 @@ -streamlit==1.22.0 -streamlit-chat==0.0.2.2 -azure-identity==1.13.0 -openai==0.27.7 \ No newline at end of file +streamlit==1.40.1 +streamlit-chat==0.1.1 +azure-identity==1.20.0 +openai==1.64.0 \ No newline at end of file From eced908ba18ee56cd41384f87830aa968599ebe9 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 17:07:31 -0500 Subject: [PATCH 095/119] Fixes --- scenarios/AksOpenAiTerraform/README.md | 50 +++---- scenarios/AksOpenAiTerraform/infra/outputs.tf | 12 +- .../AksOpenAiTerraform/infra/variables.tf | 4 - .../AksOpenAiTerraform/magic8ball/app.py | 135 ++++++++---------- .../AksOpenAiTerraform/quickstart-app.yml | 12 +- 5 files changed, 97 insertions(+), 116 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index 5d9415531..22d3438a2 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -9,53 +9,56 @@ ms.custom: innovation-engine, linux-related-content --- ## Install AKS extension - Run commands below to set up AKS extensions for Azure. - ```bash az extension add --name aks-preview az aks install-cli ``` ## Provision Resources - -Provision all infrastructure using terraform. - +Run terraform to provision all the required Azure resources ```bash +export EMAIL="ariaamini@microsoft.com" export SUBSCRIPTION_ID="b7684763-6bf2-4be5-8fdd-f9fadb0f27a1" -export EMAIL="amini5454@gmail.com" -export ARM_SUBSCRIPTION_ID=$SUBSCRIPTION_ID +export LOCATION="westus3" +export KUBERNETES_VERSION="1.30.7" +export AZURE_OPENAI_MODEL="gpt-4o-mini" +export AZURE_OPENAI_VERSION="2024-07-18" + +# Run Terraform +export TF_VAR_location=$LOCATION +export TF_VAR_kubernetes_version=$KUBERNETES_VERSION +export TF_VAR_model_name=$AZURE_OPENAI_MODEL +export TF_VAR_model_version=$AZURE_OPENAI_VERSION + +export ARM_SUBSCRIPTION_ID=$SUBSCRIPTION_ID # Used by terraform to find sub. terraform -chdir=infra init terraform -chdir=infra apply # Save outputs export RESOURCE_GROUP=$(terraform -chdir=infra output -raw resource_group_name) -export CLUSTER_NAME=$(terraform -chdir=infra output -raw cluster_name) export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform -chdir=infra output -raw workload_identity_client_id) -export ACR_NAME=$(terraform -chdir=infra output -raw acr_name) +export AZURE_OPENAI_ENDPOINT=$(terraform -chdir=infra output -raw openai_endpoint) +export ACR_LOGIN_URL=$(terraform -chdir=infra output -raw acr_login_url) +export IMAGE="$ACR_NAME.azurecr.io/magic8ball:v1" ``` # Login - Login to AKS cluster - ```bash -az aks get-credentials --admin --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --subscription $SUBSCRIPTION_ID +az aks get-credentials --admin --name AksCluster --resource-group $RESOURCE_GROUP --subscription $SUBSCRIPTION_ID ``` ## Build Dockerfile - Build app's container image - ```bash -export IMAGE="$ACR_NAME.azurecr.io/magic8ball:v1" az acr login --name $ACR_NAME docker build -t $IMAGE ./magic8ball --push ``` -# Deploy App - +# Install Helm Charts +Install Prometheus, nginx-ingress, and cert-manager ```bash helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx helm repo add jetstack https://charts.jetstack.io @@ -76,26 +79,23 @@ helm install cert-manager jetstack/cert-manager \ helm install prometheus prometheus-community/kube-prometheus-stack \ --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false +``` +# Deploy App +```bash envsubst < quickstart-app.yml | kubectl apply -f - ``` -# Wait for App to Finish - -Wait for public IP - +# Wait for public IP ```bash kubectl wait --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' ingress/magic8ball-ingress ``` # Add DNS Record - -Have DNS point to app - ```bash PUBLIC_IP=$(kubectl get ingress magic8ball-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}') az network dns record-set a add-record \ - --zone-name "contoso.com" \ + --zone-name "143252357.contoso.com" \ --resource-group $RESOURCE_GROUP \ --record-set-name magic8ball \ --ipv4-address $PUBLIC_IP diff --git a/scenarios/AksOpenAiTerraform/infra/outputs.tf b/scenarios/AksOpenAiTerraform/infra/outputs.tf index aba79ba15..29fc697ff 100644 --- a/scenarios/AksOpenAiTerraform/infra/outputs.tf +++ b/scenarios/AksOpenAiTerraform/infra/outputs.tf @@ -2,14 +2,14 @@ output "resource_group_name" { value = azurerm_resource_group.main.name } -output "cluster_name" { - value = azurerm_kubernetes_cluster.main.name -} - output "workload_identity_client_id" { value = azurerm_user_assigned_identity.workload.client_id } -output "acr_name" { - value = azurerm_container_registry.this.name +output "acr_login_url" { + value = azurerm_container_registry.this.login_server +} + +output "openai_endpoint" { + value = azurerm_cognitive_account.openai.endpoint } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/infra/variables.tf b/scenarios/AksOpenAiTerraform/infra/variables.tf index 69dfab821..05ce7856e 100644 --- a/scenarios/AksOpenAiTerraform/infra/variables.tf +++ b/scenarios/AksOpenAiTerraform/infra/variables.tf @@ -5,20 +5,16 @@ variable "resource_group_name_prefix" { variable "location" { type = string - default = "westus3" } variable "kubernetes_version" { type = string - default = "1.30.7" } variable "model_name" { type = string - default = "gpt-4o-mini" } variable "model_version" { type = string - default = "2024-07-18" } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index 52cb0a04f..db1748bad 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -1,23 +1,16 @@ # https://levelup.gitconnected.com/its-time-to-create-a-private-chatgpt-for-yourself-today-6503649e7bb6 -# -# Make sure to provide a value for the following environment variables: -# - AZURE_OPENAI_BASE (ex: https://eastus.api.cognitive.microsoft.com/) -# - AZURE_OPENAI_DEPLOYMENT -# - AZURE_OPENAI_MODEL -# - AZURE_OPENAI_VERSION import os -import time -import openai -import logging +from openai import AzureOpenAI import streamlit as st from streamlit_chat import message -from azure.identity import DefaultAzureCredential +from azure.identity import DefaultAzureCredential, get_bearer_token_provider # Environment Variables api_version = os.environ.get("AZURE_OPENAI_VERSION") -engine = os.getenv("AZURE_OPENAI_DEPLOYMENT") +deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") model = os.getenv("AZURE_OPENAI_MODEL") +endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") title = "Magic 8 Ball" text_input_label = "Pose your question and cross your fingers!" @@ -53,8 +46,14 @@ """ # Authenticate to Azure OpenAI -default_credential = DefaultAzureCredential() -openai_token = default_credential.get_token("https://cognitiveservices.azure.com/.default") +token_provider = get_bearer_token_provider( + DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default" +) +client = AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + azure_ad_token_provider=token_provider, +) # Init session_state if 'prompts' not in st.session_state: @@ -64,65 +63,6 @@ if 'past' not in st.session_state: st.session_state['past'] = [] - -def on_submit(): - # Avoid handling the event twice when clicking the Send button - chat_input = st.session_state['user'] - st.session_state['user'] = "" - if (chat_input == '' or - (len(st.session_state['past']) > 0 and chat_input == st.session_state['past'][-1])): - return - - # Save history - st.session_state['past'].append(chat_input) - - # Refresh token every 45 min - if st.session_state.get('openai_token') and st.session_state['openai_token'].expires_on < int(time.time()) - 45 * 60: - st.session_state['openai_token'] = default_credential.get_token("https://cognitiveservices.azure.com/.default") - openai.api_key = st.session_state['openai_token'].token - - # Generate API response - st.session_state['prompts'].append({"role": "user", "content": chat_input}) - completion = openai.ChatCompletion.create( - engine = engine, - model = model, - messages = st.session_state['prompts'], - temperature = temperature, - ) - message = completion.choices[0].message.content - st.session_state['generated'].append(message) - st.session_state['prompts'].append({"role": "assistant", "content": message}) - - -def reset(): - st.session_state['prompts'] = [{"role": "system", "content": system}] - st.session_state['past'] = [] - st.session_state['generated'] = [] - st.session_state['user'] = "" - - -# Row 1 -col1, col2 = st.columns([1, 7]) -with col1: # Robot icon - st.image(image = os.path.join("icons", image_file_name), width = image_width) -with col2: # Title - st.title(title) - -# Row 2 -col3, col4, col5 = st.columns([7, 1, 1]) -with col3: # Text box - user_input = st.text_input(text_input_label, key = "user", on_change = on_submit) -with col4: # 'Send' Button - st.button(label = "Send") -with col5: # 'New' Button - st.button(label = "New", on_click = reset) - -if st.session_state['generated']: - for i in range(len(st.session_state['generated']) - 1, -1, -1): - message(st.session_state['past'][i], is_user = True, key = str(i) + '_user', avatar_style = "fun-emoji", seed = "Nala") - message(st.session_state['generated'][i], key = str(i), avatar_style = "bottts", seed = "Fluffy") - - # Customize Streamlit UI using CSS st.markdown(""" -""") \ No newline at end of file +""") + + +def on_submit(): + # Avoid handling the event twice when clicking the Send button + chat_input = st.session_state['user'] + st.session_state['user'] = "" + if (chat_input == '' or + (len(st.session_state['past']) > 0 and chat_input == st.session_state['past'][-1])): + return + # Call API + st.session_state['prompts'].append({"role": "user", "content": chat_input}) + completion = client.chat.completions.create( + model = model, + messages = st.session_state['prompts'], + temperature = temperature, + ) + message = completion.choices[0].message.content + st.session_state['past'].append(chat_input) # Save history + st.session_state['generated'].append(message) + st.session_state['prompts'].append({"role": "assistant", "content": message}) + + +def reset(): + st.session_state['prompts'] = [{"role": "system", "content": system}] + st.session_state['past'] = [] + st.session_state['generated'] = [] + st.session_state['user'] = "" + + +# Row 1 +col1, col2 = st.columns([1, 7]) +with col1: # Robot icon + st.image(image = os.path.join(os.path.dirname(__file__), "icons", image_file_name), width = image_width) +with col2: # Title + st.title(title) + +# Row 2 +col3, col4, col5 = st.columns([7, 1, 1]) +with col3: # Text box + user_input = st.text_input(text_input_label, key = "user", on_change = on_submit) +with col4: # 'Send' Button + st.button(label = "Send") +with col5: # 'New' Button + st.button(label = "New", on_click = reset) + +if st.session_state['generated']: + for i in range(len(st.session_state['generated']) - 1, -1, -1): + message(st.session_state['past'][i], is_user = True, key = str(i) + '_user', avatar_style = "fun-emoji", seed = "Nala") + message(st.session_state['generated'][i], key = str(i), avatar_style = "bottts", seed = "Fluffy") \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index 4564df825..7238c5ca1 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -3,14 +3,10 @@ kind: ConfigMap metadata: name: magic8ball-configmap data: - TITLE: "Magic 8 Ball" - LABEL: "Pose your question and cross your fingers!" - TEMPERATURE: "0.9" - IMAGE_WIDTH: "80" - AZURE_OPENAI_TYPE: azure_ad - AZURE_OPENAI_BASE: https://myopenai.openai.azure.com/ - AZURE_OPENAI_MODEL: gpt-4o-mini - AZURE_OPENAI_DEPLOYMENT: gpt-4o-mini + AZURE_OPENAI_ENDPOINT: $AZURE_OPENAI_ENDPOINT + AZURE_OPENAI_MODEL: $AZURE_OPENAI_MODEL + AZURE_OPENAI_DEPLOYMENT: $AZURE_OPENAI_MODEL + AZURE_OPENAI_VERSION: $AZURE_OPENAI_VERSION --- apiVersion: apps/v1 kind: Deployment From bf975d85af195eb2cf352305725c47839db1fadc Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 21:10:32 -0500 Subject: [PATCH 096/119] Remove certmanager and nginx --- scenarios/AksOpenAiTerraform/README.md | 58 +++---------------- .../AksOpenAiTerraform/quickstart-app.yml | 43 +------------- 2 files changed, 12 insertions(+), 89 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index 22d3438a2..8ea7e3fc1 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -8,30 +8,24 @@ ms.author: ariaamini ms.custom: innovation-engine, linux-related-content --- -## Install AKS extension -Run commands below to set up AKS extensions for Azure. -```bash -az extension add --name aks-preview -az aks install-cli -``` - ## Provision Resources Run terraform to provision all the required Azure resources ```bash +# DELETE export EMAIL="ariaamini@microsoft.com" export SUBSCRIPTION_ID="b7684763-6bf2-4be5-8fdd-f9fadb0f27a1" +# Define input vars export LOCATION="westus3" export KUBERNETES_VERSION="1.30.7" export AZURE_OPENAI_MODEL="gpt-4o-mini" export AZURE_OPENAI_VERSION="2024-07-18" # Run Terraform -export TF_VAR_location=$LOCATION +export TF_VAR_location=$LOCATION # $TF_VAR_example_name will be read as var example_name by terraform. export TF_VAR_kubernetes_version=$KUBERNETES_VERSION export TF_VAR_model_name=$AZURE_OPENAI_MODEL export TF_VAR_model_version=$AZURE_OPENAI_VERSION - export ARM_SUBSCRIPTION_ID=$SUBSCRIPTION_ID # Used by terraform to find sub. terraform -chdir=infra init terraform -chdir=infra apply @@ -41,46 +35,20 @@ export RESOURCE_GROUP=$(terraform -chdir=infra output -raw resource_group_name) export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform -chdir=infra output -raw workload_identity_client_id) export AZURE_OPENAI_ENDPOINT=$(terraform -chdir=infra output -raw openai_endpoint) export ACR_LOGIN_URL=$(terraform -chdir=infra output -raw acr_login_url) -export IMAGE="$ACR_NAME.azurecr.io/magic8ball:v1" +export IMAGE="$ACR_LOGIN_URL/magic8ball:v1" ``` -# Login -Login to AKS cluster +# Login to AKS ```bash az aks get-credentials --admin --name AksCluster --resource-group $RESOURCE_GROUP --subscription $SUBSCRIPTION_ID ``` ## Build Dockerfile -Build app's container image ```bash -az acr login --name $ACR_NAME +az acr login --name $ACR_LOGIN_URL docker build -t $IMAGE ./magic8ball --push ``` -# Install Helm Charts -Install Prometheus, nginx-ingress, and cert-manager -```bash -helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx -helm repo add jetstack https://charts.jetstack.io -helm repo add prometheus-community https://prometheus-community.github.io/helm-charts -helm repo update - -helm install ingress-nginx ingress-nginx/ingress-nginx \ - --set controller.replicaCount=2 \ - --set controller.nodeSelector."kubernetes\.io/os"=linux \ - --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ - --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz \ - --set controller.metrics.enabled=true \ - --set controller.metrics.serviceMonitor.enabled=true \ - --set controller.metrics.serviceMonitor.additionalLabels.release="prometheus" -helm install cert-manager jetstack/cert-manager \ - --set crds.enabled=true \ - --set nodeSelector."kubernetes\.io/os"=linux -helm install prometheus prometheus-community/kube-prometheus-stack \ - --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ - --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false -``` - # Deploy App ```bash envsubst < quickstart-app.yml | kubectl apply -f - @@ -88,15 +56,7 @@ envsubst < quickstart-app.yml | kubectl apply -f - # Wait for public IP ```bash -kubectl wait --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' ingress/magic8ball-ingress -``` - -# Add DNS Record -```bash -PUBLIC_IP=$(kubectl get ingress magic8ball-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -az network dns record-set a add-record \ - --zone-name "143252357.contoso.com" \ - --resource-group $RESOURCE_GROUP \ - --record-set-name magic8ball \ - --ipv4-address $PUBLIC_IP +kubectl wait --for=jsonpath="{.status.loadBalancer.ingress[0].ip}" service/magic8ball-service +PUBLIC_IP=$(kubectl get service/magic8ball-service -o=jsonpath="{.status.loadBalancer.ingress[0].ip}") +echo "Connect to app: $PUBLIC_IP" ``` \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index 7238c5ca1..668267a70 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -29,6 +29,7 @@ spec: containers: - name: magic8ball image: $IMAGE + imagePullPolicy: Always ports: - containerPort: 8501 envFrom: @@ -38,54 +39,16 @@ spec: apiVersion: v1 kind: Service metadata: - name: magic8ball + name: magic8ball-service spec: selector: app.kubernetes.io/name: magic8ball - type: ClusterIP + type: LoadBalancer ports: - protocol: TCP port: 80 targetPort: 8501 --- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: magic8ball-ingress - annotations: - cert-manager.io/cluster-issuer: letsencrypt-dev -spec: - ingressClassName: nginx - tls: - - hosts: - - magic8ball.contoso.com - secretName: tls-secret - rules: - - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: magic8ball - port: - number: 80 ---- -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer -metadata: - name: letsencrypt-dev -spec: - acme: - server: https://acme-v02.api.letsencrypt.org/directory - email: $EMAIL - privateKeySecretRef: - name: tls-secret - solvers: - - http01: - ingress: - ingressClassName: nginx ---- apiVersion: v1 kind: ServiceAccount metadata: From c40b313c8b0d2fbd295b8ffa5b9e7a75ba72487c Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 26 Feb 2025 21:10:50 -0500 Subject: [PATCH 097/119] clean --- .../AksOpenAiTerraform/magic8ball/app.py | 215 +++++------------- .../magic8ball/icons/magic8ball.png | Bin 37452 -> 0 bytes .../magic8ball/icons/robot.png | Bin 1686 -> 0 bytes .../magic8ball/requirements.txt | 1 - 4 files changed, 56 insertions(+), 160 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/magic8ball/icons/magic8ball.png delete mode 100644 scenarios/AksOpenAiTerraform/magic8ball/icons/robot.png diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index db1748bad..712f1f26d 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -3,21 +3,30 @@ import os from openai import AzureOpenAI import streamlit as st -from streamlit_chat import message from azure.identity import DefaultAzureCredential, get_bearer_token_provider -# Environment Variables -api_version = os.environ.get("AZURE_OPENAI_VERSION") deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") -model = os.getenv("AZURE_OPENAI_MODEL") -endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") - -title = "Magic 8 Ball" -text_input_label = "Pose your question and cross your fingers!" -image_file_name = "magic8ball.png" -image_width = 80 -temperature = 0.9 -system = """ +api_version = os.environ.get("AZURE_OPENAI_VERSION") +azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") + +client = AzureOpenAI( + api_version=api_version, + azure_endpoint=azure_endpoint, + azure_ad_token_provider=get_bearer_token_provider( + DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default" + ), +) + + +def call_api(messages): + completion = client.chat.completions.create( + model=deployment, + messages=messages, + ) + return completion.choices[0].message.content + + +assistant_prompt = """ You are the infamous Magic 8 Ball. You need to randomly reply to any question with one of the following answers: - It is certain. @@ -45,150 +54,38 @@ For mor information, see https://en.wikipedia.org/wiki/Magic_8_Ball """ -# Authenticate to Azure OpenAI -token_provider = get_bearer_token_provider( - DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default" -) -client = AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - azure_ad_token_provider=token_provider, -) - -# Init session_state -if 'prompts' not in st.session_state: - st.session_state['prompts'] = [{"role": "system", "content": system}] -if 'generated' not in st.session_state: - st.session_state['generated'] = [] -if 'past' not in st.session_state: - st.session_state['past'] = [] - -# Customize Streamlit UI using CSS -st.markdown(""" - -""") - - -def on_submit(): - # Avoid handling the event twice when clicking the Send button - chat_input = st.session_state['user'] - st.session_state['user'] = "" - if (chat_input == '' or - (len(st.session_state['past']) > 0 and chat_input == st.session_state['past'][-1])): - return - # Call API - st.session_state['prompts'].append({"role": "user", "content": chat_input}) - completion = client.chat.completions.create( - model = model, - messages = st.session_state['prompts'], - temperature = temperature, - ) - message = completion.choices[0].message.content - st.session_state['past'].append(chat_input) # Save history - st.session_state['generated'].append(message) - st.session_state['prompts'].append({"role": "assistant", "content": message}) - - -def reset(): - st.session_state['prompts'] = [{"role": "system", "content": system}] - st.session_state['past'] = [] - st.session_state['generated'] = [] - st.session_state['user'] = "" - - -# Row 1 -col1, col2 = st.columns([1, 7]) -with col1: # Robot icon - st.image(image = os.path.join(os.path.dirname(__file__), "icons", image_file_name), width = image_width) -with col2: # Title - st.title(title) - -# Row 2 -col3, col4, col5 = st.columns([7, 1, 1]) -with col3: # Text box - user_input = st.text_input(text_input_label, key = "user", on_change = on_submit) -with col4: # 'Send' Button - st.button(label = "Send") -with col5: # 'New' Button - st.button(label = "New", on_click = reset) - -if st.session_state['generated']: - for i in range(len(st.session_state['generated']) - 1, -1, -1): - message(st.session_state['past'][i], is_user = True, key = str(i) + '_user', avatar_style = "fun-emoji", seed = "Nala") - message(st.session_state['generated'][i], key = str(i), avatar_style = "bottts", seed = "Fluffy") \ No newline at end of file +# Init state +if "messages" not in st.session_state: + st.session_state.messages = [{"role": "system", "content": assistant_prompt}] +if "disabled" not in st.session_state: + st.session_state.disabled = False + +st.title("Magic 8 Ball") +for message in st.session_state.messages[1:]: # Print previous messages + with st.chat_message(message["role"]): + st.markdown(message["content"]) + + +def disable_chat(): + st.session_state.disabled = True + + +if prompt := st.chat_input( + "Ask your question", on_submit=disable_chat, disabled=st.session_state.disabled +): + # Print Question + st.session_state.messages.append({"role": "user", "content": prompt}) + with st.chat_message("user"): + st.write(prompt) + + # Print Response + response = None + with st.spinner("Loading response..."): # Loading indicator + response = call_api(st.session_state.messages) + st.session_state.messages.append({"role": "assistant", "content": response}) + with st.chat_message("assistant"): + st.write(response) + + # Re-enable textbox + st.session_state.disabled = False + st.rerun() diff --git a/scenarios/AksOpenAiTerraform/magic8ball/icons/magic8ball.png b/scenarios/AksOpenAiTerraform/magic8ball/icons/magic8ball.png deleted file mode 100644 index cd53753774ed4e666c7093f6d58ca02a25be36a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 37452 zcmW(+1yEaUv&IP?+#$HTI|O$v?ozC1ahKrkTHIRPiWDzyrC5s=_YW@)z30D~$s}`T zk~x#?yZhM2X=x~8p_8G*!NFlEE6M4=!NIft?+HQxuG~?R(gH8+Uh;-svYvKUUXE_A z5Isj1J2(YTM+mPVMBU0C!pqIe2jLbG<>MFS=Yg|VxholcgoDGt``;6Oek0@sxQJvgqb>sn*PMd!VucI`msYMUC!^=LdKUcn zG3%HAzpmfP6w;t@`nG0JGYB4G`n_15u+NT9e&GQEGi-!PDoUaMy*K!6^#-|!cBFAl#g zs~yWDLx!VYXUHQY#NmJ_eHYN0BxA^V0Iw}hJ{wqC44>j`d?-vW2Qb~WRivrLO z$;rsb$jKk;APW+HI@$mG79HZjM2I~rNB{H21(mp8*q=XdcO9V*->Kg+iL7yug@<3^ z(%JC-=0Ww5QeTj`AG>}8pWhV!dm)~0;bjvxFDAm$UhiN3bi2Z;DQ$R@Zi_rD!N-T# z6MQ|vG_ZCN`Y#ZElY0KTx{loz`S&jTpYJv@lvh>#I+uj;as*Vn#ACx1kedXHmmR(z z;*o!rY7IK7kf{SACMsE?m~;u+nGwjfE`R_2{qXR>)mJ)& zoK8$k93MM|Vby#K{r5KJSbBuNfj2Tl5=97+fgiCUC?ZCSMr>4wucc*`dWXz(B@g{% zzehz|?8f*jH3hGbAhA}z9)aNW^fY{;s)`Y$yp21xv7e&sk1#YslrG79yak6#Vzvc#c(E}qLB!8{nYtm^ zcxh;0J&9gLh{wXh;)eWcF#BfpJvuoVyG%Rr2UBGWP!WVgx8yK1>=G9ZH8r(rK~w4nK zFp(8>Dw%`QkICGgCWu-8K_~ zexOksPIg3c)3E=zg5g)+1!xZ5?3Vb_`#d{%UK;Y1%OrB4^OaWAoXaAnq9t*l6l~;& zF}yT#g9bLIr!g-Xvk9w-q$5hCln_%P!w6xhxbSC=^s)X`Vy07^GF5&mZD!R>NA0&C zlVp?Be9^Tp9W%|H{3Z8g{zDy-a1a=#n^U4elP*i@Q4}+{N}vi;I#Wqd-ao|WSD3^H zKVc|;3<@gzT~JVfc;|58AxZg)QjVSKtfXpVLzgTQ%gey*lQT|Bc>&$FiT`dfJVxix z>^`NKI22bDlR9aV@3fcR_a)0v)p(NON$=RT@|ienn5!H4K8zF|R;-GV{Fwj_`2S44 zekeV89(M4W!UwNLk?kHGF;`HlNybXft+*{9=pd4YFW`qjxRj&}1NbxG#6@Y~F|`lt zy2P-QFtj%<`GZ-!z`S(o5VBcks}u4%zOpg&5awlOa(MYBrr8MIld#by!+z$b7%LvA z$e5e6GaJ+U?7%|mGN2^?8H^vet#^9d9L!8iC>sVt*m?FG9_P?xc2QM`XLHq@2~%n& zl=Kw?yGLLic4d&;zPX1*?XDGJ+FuDm1~n}p?E0HUzMoD5DSTxQ`-#le6pt7bP}cb< z#Lpm#uN@r=G;QlPZ(5m5kQWP(Oi*|_3My($baY9??2jJ{h=Vu7RKZsd$QIRsS`lB` zZ)DrFwB^*{v|j?0AEKxxQtZvH$0rIuDa0||Q57U&%lz8t`WO>7YynA<`O-!OFJE&$ zu$|o|&Ov!_uu-ns)%d|sBIEI{SbWZYcB&KlgkQl5vIkXP0i1lepmCA`l#q}1{ z3k1D^hnLqJ^|$r5rVsnA7|`fA)4m{51aLmnD`#d1|4E5^gj_&CfO~31ACYu&2W~O% z$5dNw7G4=mX=BfMc|AF1qa1dmlfb?7sD{{*)}}8%0zI5JJ{11;Frz;C55}yg+b|ko z37b?=e(=&C*b=V%xv_yI9rC`KUoruLBmn1#h=>>*9Q^(J_Y#~*AqdzDkrm!;^Q#N_ z8Icy+JN9(q6vx9Y3C!noG|{{#d?8B^%-nD^s^b2O0j0o*Qmgd&NYGx?IV)EA@<-6O zpUu2y%rE=XlHCZD$bv#VJn;Nwnk1CTwJ+es=exGB{q3CXz$tY)RbxpLh^^f3$D2c- zP6qQh^EqMYHSnhXK*h{u%FQ{hj0g+-9JcK(hUt^H=MM_2l0ro*u4I@iGF9q(xP7RF z;6G}I{KZg*yHN<&DoW4DhBiE_E7lZbzQ#=?@!ggncr*UJ8qh*|K0P~w)j#CvXi2d_ zY`5NphDvP-2?%)KpKtuBMz#&Fg+$FbmTr$hNN~dfoYrhL^bm9w3bl%|Ll`@X7asPq zU|)~F8X9DTt&$GCqln=|s7FoGgkAs6*W)+({RO*Zgo{8Aq|hXViT=i0LByN!m6qLo zTy_hL&Pr*S672sXqvY@X->QJdD|`rEOX63=kgYxvw?qjg_!q;SDcJDH$Pkp2WPU#p zu_zVh;XxAaP=?fk#@xnO7o?!4dpam=+N;@Yh)-f}tD~si0($3Aof=O<*}!@*i8pIs zXVD)rmQ2~(+l%>R;!69)#eZ4`RDo5E3<8Rk@udU<-hjkWbP|3jIvRy%o1JZzS$>gi z!PGYvC6O(8PeWJ5`&QEKxMO*(hl>Y2X45zNe|PKe%OjFv40 zMaj6po2e}(bAw3-_A-3p$3)y&sZp(s8Bc`vT5=D2V6N(csn!$Ut(9wr5r}+9AdUic^k#s@xQ2h6# zoco=*IkbYM1z#dRp}m3LA!}uF>!Bd-V?`lQ?zg`hgpmvkR`29u1*a% z_DG;a!zfL_#j~t;vT3{&xhPk3v4=Esb33D1qmox#mP3mX7A7-6-sjIOi8=JUrFuw= zR&(SI%T4_Auk&! zI+35@?b$~iQ#C_^aphetrwN^gt?T9CCwE$gflr>CdSHooPYR_7F;5Wl1H2U~MP;80Oeju0x#k!4lk zK@TqLow&QhhNP0#QIrLXWc;Kq{w_mcgi6@aQf9i;lLT-ojy5HZCHMO(*pU=Y+}cIo z>MMj_)TxzT5dV1+;X3RkTI=fKYmLq65{!oKq8O7tL`|ZLg#vxdRQE16Tag4tf(W@h zTWOVs+ywDXBOeaR!S~mjbO;Ix>L6#Gb<>`<`RrkFX5d8a6?sQ`90^p2<_B4i!Ahrc z{N&s2joiw2mbfLiqTaIWqC5V0DRjxjDHb8|cHKqv;6W-PlMYF_WXch>V5LeItL27N z@4#jEjb`6aNR^10bveL}00sz0UtL;i2M%V#o5rFsDyg5_d3G%SB~=0Qd?F6o=8uu$ zc)bY>zWbHAmN@Y+mJY|Xdy^4ldPI*T-e2%iAX#qR7tc~sVJ_K$yuD<|@I^QxD)-Km zdTv&xA|A{#{_saJD#&f4J1T7;OlA!UWVH$(qL@PE;_5mIAr&~awBv~nn^Zl2a?;@C zzL^u9Zw7fPUqC zw(>`xsz^ANf6=)$X&84G_YM9>>W05JP>W)6fwLsl=J2~7uU%J}00CPy4W#Dz%Ea-H z-eekWALR7&?e#ej7`BI=3y!4w+6Xkn&cz}hQ4}D=zf|7Skc0!hHU@_p1tAYXPcU%K z=oQZbE(ng4^54PgnG1x;ns22a=@vRxbUgs8F~@F=Vx0urRX)&i>m(t zYvE+3ljRXOYYgBWh{};OU)kRB2$ZZpoYJw%u+3~Y^Y$j6t)iyoT zyeZX18N=;tEc#*aS& zNe1V3!ejlI#vZ#95fxkdgX9b24jmknWXJwT5hVqM?VU49d1)Bkxh4hyj*3(w)-UCw zAp9Yf40j%F@7cYkNi_mKMpTZ2Al6y-L>$Ms-B}MYks%q{zxU?x_qH{#d^+<5`kC4k zi=jl3PVU=bnpx?m?Hp_okn(!;4Y!V{m>Bi-N4=Uq=1>ii^S#_8PI^}M z=p|yu7QZpkk~c#7u`DOCZnyAv!ur;$Y0<9(qoPxnepV2o-`<~zx)k)Zz$ z>iRoW%s*t-RdnQSqJQJ+pt?0j56-^jt9FpYy`p4DH)>QD7@+gZT>lR+2 z2G7a7=|m2DbmvSNE{UUYMJSZVql&;a2`A8XW<#~K;iHLF{Ee$NRf9!86)WB}QCKP~ z&aRuK1hmo`Bz#7kI;Vr{xp_So4wmqf;3zLI_k!WUj^?VWs_ct{?*_sVk38;JsFe$? zKM85m9Q3YKiU={}p)F(w3LsRsq!N7&eXE*~>H1k+>ej{N?e@)rOGG*?+@`q&=cB?2 zAVSc}(fT4Y5umA#x|w!+#c?jgxSZ^4BW%Au|F6p~|N71WpZQ+0d$-=m-9 zbI@Y{1JZTI6!*5Cg24GXW~FpaDgA^7PTh0BhMrebv@>|mJ=jgq_b-Oe8z9PBgR{bR$y1^x7xgOcLL z+XPr&ChTTsXUoQ5d;9u+)J`w5_$qjmu*(e`60O!Mb+rCG0K zjw-e4vJ3Y9h58#x$G%0Nqmzyw`zg;Lwc~dW4@RQ;UTk6^@tTs0qLeF`c?!WZeA(kZ z>>?<$#Fw^I2Iq56C%QS-^{^f?=$@HkO~~R$sZ(gX{^&%+?UwSIs5d`~!n2s9jItY) z+@tA5F|}!ST4F7-!ep&(?7OYxlJPZj9s2j@S}P2Eb~2ho4m3qdDysL)n9@o-v3-$5 z=&$gv<96XkG7}x|;6du2t$eurJ9t`51zdv-p+8N|4^`%SuS$Qpw4=eX&)-dgNj}?F z@X5NxN~PEA(Q&?xPd8DqmnDBz_s>c!;Cr*Y>V)rJipC<(&bs^8Ax;3Hprnk#q#z?J zV~UPv9)>)fj;GNzRSY&Gg)?`m%;r$Tp1z=^>b>abh@%lbWxuO@5Tko4y&9Z;HoH$> z(s5gx*%>&luf9)ME>I_MEPY}cdEeh-vCgNKG$>AdW;pr%>FJ4nN!5c&59!3(5=$fG zVLK)@=7;zV=DCr^Nhpal2&qWe!#X6S=<(+qR~Yp>kVEO#JGqjx?}fJTE2C+!IL%{T zY-GO#fvK~ZzXU5krlYKn(nH=clk0@|?e!Bt{gk;$sJZbhma~$ zs%@>c^TBe;y(sTz@}kn`zMbib3?`4Y?e8%;23%(E!J9q)H@C+Ne0+SZt*z_Dm}aBx zxMpT%O+|oOL4)IHSP=FZzq)DMA_GyIQ91(GnCm_r&Cj8S(5E2pMyKL!b9JvDQ|i7+q&$W0e?V8$4NoVGY<9X zY8+jbIQ#rr)%<$-kDPx}VrIu%>DsNuy$)zJpBuWn-)qYeycjjn^Q+ECmbL7A3O4c! z@&YP^iIFk44JG==d}^;rHWqAmcbAyU9L6~rg$qY@vsCW;)$I>9R%vT%#zFCiP&P=L z(j-iAiY(BVuy~P)NhVOLo-1necv+YD^lDQ2NPjdFaVY%{>&mt{cjgHDWbovg&!vI*9ZXK@Dm4w?&(ylGQvIk*(rsceF49AilRrqRsK}5@{YIz8(*T%j> z5_0|`6Fg`iVd+YI+DHKwgfb1*5iS+TOCEj*^tEBIZHIMpRr#P^X1$B885 ze zGpa=_KOFA_iz8Hn-qXqzwq7#|&F=stq$y5Mcs{Q*WAhyA9lIp+X7iLT+f}}&gwTJo zB;h5r?m^5Jn9m0hmhPd2UH&(W_u)=1E@mon>;fPfxJJ`11ZP8^+U!Dt|Ni^$`Tj!D z7cObugV6-S6<2VJSxWU4)P4-VFm2fKsI70Wc48FRoDHslcDikw=0Bq3(O=s;y1=TXG%RJ=8&rd z3tJsQcb_n<;n78_P9!H-K8kXzVm7X+4J+-#5e0&4F#meh{8FrmbZdJxyv^K+;K{&=2g5!z8^&vt;H5R6`?vI*x;Hd$ zn;MQ%l!I*<{qHx67J)`Re}$ZQC%T8ZwMBl|fV%11JvytS)cAIC*Z+OTks5YMp#jkx zx&7&zDXd>tN_A}g`C#_V=f9QKnB#dhvLlEj30IPnebrUztVDKTT^sS!?6}6_d5mYo z0upF3xIf3)uYl-FXiqw!h3B`$UfXwddy$)rF-0nc%!TP$9{(6gZT4&Bs1RaGglfu5 zgg_{TsB-r2-*Z~@VU7@f$@?5^L<=_W@oVny|L1BfY$mGJoHk7q zai69&j)WRHFx=`Diy?dY0`uxD{cI%@!QNEP??xKC> zwxYvgkGK0Dw+-beYJP>?)LDNWGMjW35Ur0v(4c{LtyI#>hrMo9F(A zy1SCsP}bKMaKt=qp}`tq)@BpZuW#6(HV!OM%_7LCT}H0wbCbsM0U zz;MaEjb~+!BKn`w61uBwL90jp&*VCNB8DG7T6gc2QSIYZ z)|fwY`}&jWL|motG;Pi`gnPa>?dcnm4R2IgLqjqTKX1;szq}U9rHGgq#$T*^@oNY$ zD;R~*4bY(LOvl(MNPF2FCO#LN8>_4FH;d3Nx7wrg&5*WaS*3(6t=YiVMMDdID@#R` zwMteu26f2~nSb>vdLK`zY0uMN@k3b`$r-q^>M8`1&vl=oyk>WaR~fxDP*iz8v$!y% z(h}cF2S-wMxNdaM&d#n{ReXiOFew1-si44SP4ex*hAgnKY+JG@ac$Hhk3Tf_TP7ZY}89(57&rW^Rh!8s{>30vi zLk!v2*l?h#DAOiAV;Bw>UZk}D@5??VN9`Eh)}l~u5W(*9aPQv!@TRJHn;#b`wNW6= z$*bv0I!lxZK5{h1uQ@jj>3|qs^M7-?jOfOkI9tZ(UmRP~&}wii9=h7#lI9Nb7j#hm z=IA=0Rpy_9bE*CP?QL5}$Ia}I(6<{*^9&O0j!-imVDM3d`8$qK z3f|9b@}0veu4iqkRuKqqxJ7|xRTCK1st7H3vE#2T(7i+KF6v6s^>J=6nXxV-#luKm zpvJ;3qm>AZGs!Yna85|rfM~K@XN1^n<)r0=Jk)Cc!~C01QftKRn7b>%D2?BAsceie z;&-s53lbx1c>Y?Kt6TTYh3jO%Zux5L2M_A2hVbhM`oqff_0?`Zxw^D^slQmIPcVAr z3YyJi?--U4mU*YsqB-tkyot)K6UhKIH2GLD4THIlGMX%w%4})Mt~44D5h@07GN8p1 zf6B|VvQ31=dHx$%Ue-5^oP)2Mjw+>zJCk(uc8*o-#=lJ)%vhscbu+!ohm$c8@SsM! z3hBr@|1;cy*3co;(@PLjNNlE0;9Bu^{s^v6Cn$6si|*S;?pb4MLFMj%Kd=_fq)#%@ z&9NGcUh@ zpjs__g(zY?*Wh$%LB$R>mF2-Tj;k*L7W?%TYpVLMBFmc9$U~`)`srS2a5m0I&13IO z^K)}aC@6w{%P7ech)2r&UYCUy_B*Trz8oM2uu~3J1;~&n8po=iR7!91U~4PD-Ua@; z9*g;-%9EywyJcj7DdIsJ8DarROuT=9H6?OR<%B-|AaoN*gE6unZo{Q7TVK z<&BhtHtcSjJ9i>oO9tMZ0Xowc#N0MLYM7Rs6>6CT&iv5m++&y3ca&I}wuSU?^LYJr%P;)2ETW~x@?NTH^S9JTtsB$&!2vu|5jNw z{utoH0G|XfNdQWT?2BW*%iTU!&+P4UV9FetafMwA81HFfaXEfj$5iN_{s zImPDoghmn6k$b7I1?zkr&*WQe{*bA3+whv!pUs{$Ow%a~cG7SV?s@$%1Xl~EjcX1j zWfU}Bj#X`tYiQUi{3@LpP0Q&)0JOYzkKNH2-2b6eZsc%j^j-kG?1pLW1z~KBNn`=- z$&Qix2hp-QQgjuTI)SGLG!DWoIe*S@k*Ww%`nO~tUvedVSC=HTz+9lXt6Ploc-~7% ze#qz{H*zr{g5=7zU#D5atH2?Ux%RvFM@5t%LD-n-6I?}I>$kvr&P*MU>~H<7p~@OP zeAv1=S{BOMRyxCZ*7q9z+z2g2mVJ2D(JvVNarfUmlx9u~-Ni)Tp!c>D+ZKBay6ApY zxk3}>HTtf{V1!`AD_LJik|?L@@nk1A@h&)9qE0?<^o&L?F5+XIS=bZlCaaF0qFyBzEtqjEkKZ|iNH`4U|xBu zq-L=0m=3`kDygbsxQ~xI=uDxB+&7Bp;a2ST*iw*~AYM=^btR=PAjGUzxwn1}hFm(v^ zc7$j-qorT;EGDKy`aaigAXFQDG))6{+OWev>Py_UT+V;d*8vmZ&QcjRWgRQENHpz& zFZ;|T3(}W>e1d}kqCqT8uukN8=~u-dR&bvp*OE3dS*ubn7%)qpM$jmv=XAZ#W1haQ zz~uNJOz_0+*Fa<%QgHR@l+8{Xrjg3tVliR z04XIRpShpsiU6X?<-qE8}N$REtJ7UJv6z`3;0%N+5M6 z9SwbbMW=S1x)I_9HAXh!**Vea*2R;$My20UoIsLTl!OPOoe#sq!+U%5aLR-V>$k?l zcHfR71@h(~X8F^a1rI*hUiUGV$!5&7zj_A=F4k~kG7Rtn^1Nd!x>h7|id17+ZtC4f zBN9ReeoDXKAcML(8rTbo6SB#09h?FB2SLb>P|n`_?y2Vg?K6T0aSqG_9MDYJ7&qaN z>BXWz9A~J5G{?(cA@)T`P!Pz15Cum>cDK2UA!SS`rxm7Ojrt! zH>0Zr20q{Yw{|Q=u8hHrmQi%0P}g-9z$S7LdqV$CuAcNK=(p(svu4zJqV%Hh*(jw2 z@igPWm@Hv=I_>kNhaJ8KFEMF7;z{sPhs{`W@Ug0eF=hl226WLfARvD|&772?yo`b) zthJ+qxSRPAotTYb76oUMs*a>Mt!ZLzz(61Wk->Nz(!Q3xsKyLEL^_dvrKF^snVBI` zif8$caPD&w8Y{uZ+75QJMzjLc#A3*Mm!-qcr%?FOBe=kUZ6N*w_`~p5%xvN#?qCrX z;|_sm>O-%m-@Nlscl0i&jkjMHw+;Ph5{B+~+L>!!4Zhkc6DhfJxe{TB?#)v^++`^^cOLmAXkHvPs5!t!Y977R^-taoJeG= z$p=zGTH`Y3Y68yQbTLMz$va*isylmJL?cVjvF0=!0t8RHBp%&0)z#JCzLBr~8W@na zu=oSW8CKTZ!x157Du@%~>^+u)9Y&wxXj%Ccp5Dekoxhe=*Nt(cV~h+>&FrPO$n(Qb zT;k-B4HErnKc`Nc}pd{wbAg^=Ys{^Q7$7lkzWs?%2j*v8j;(nze#3OO&UYe&Q+<#eCvQya9 z(cKqBdz-_@+`Fx@O-Hfc#;ZYhfYog_+l*~(LEzAsdGEk8GOr`S`4wLqq*pNr2ct1m z@!wZ%lHbb6tw(KsK)}GLr=`?LwwZqy)%Hd0WdGo{S4sXLVqgYPP2~FJ_0s)5Q@|Vu zGah+?D1cH@6uHtRW~|%m)8F(3^MiaS5;1(WZ%t47mpz7EOO`{1-N;(!p>T7nzcNoi zZ!3KS5>!KmWcw=K?USh#3AOpzm;`l`yZWz>W+BFM%rTMMX&*37Ftg#iuvYv4KfTF6 z_qA{t_Yi&L**q$Mx5C1YMMlcHa7F2{;i~!l{T1l?FWtIFEIN~Y(Rh#WV8YEq zrRUgMgK3xN_1;9eVoH9zN)$vcNwZKG{u%ziL-U(9lw$&h&)a^0*8x$y>WV9%X(153 zK1FX3|Do`vP_z^(@06$Jz4zO_k$Ht95~I1)9s00vi72qJ_Swp^OyBe^{qWuLJL07w zZ2qugX=o{JuiaF(Y7VE2V(-U8?B~IBd6LnbnXh#; z%lE3{t<$p$8NH8Rp(lB*udfFq z5P_03Des-4)8K2D88&Lb<3_f8f_YO!|BwXSl~P7W6+qwsNghki5rgG50S?@5c$ZQd zOEzw{)wQ+33q?gmfQG7u54gN0=4WS0UMa|Qnf;dqO;bIGbcf373cFtM`rb3NVRm8D zOhC&)(1zLpcqK~X?XHi9MZtF~=+Pxmu!g4Q4`8~{Mrn(ia?{}B)G%Z=cXx|A2aG+` z6py?sR(s@55Z^cFl{7y!Pge$!ozhE^g%plf!maoCf3&x^*VSDB1VJ)5oqutTniT>TpjKgV zvArBsUSqq$GNqhE3*$gqJ?gHS$UlbNa`+6Kj2#iWclb&C3N^S|_;Lo)60q0xwKWr0 zDFY}E_*)YS%aa3q%l@J~gEXmi_-PvP(8OB3Yy4@sy8b8ZAh2VkYf6W z6HHX#I6(OZUBjej|3l}^egRdy=Z?lJP3*22>voHwGIlWNKXlusZ~G-)?% z7YQyd%+HUo8h}&Mi&`sCwv>FNwM7_>)8;)Fb=}Y{9iT-Uy7hV0nH@jn##QKWzaXoz z2TsRYw>O)TF0HutaGw9L2lMB-uE|;3(*Wih8p0H6sgS=X@6golp=|wi2_H|uKawnX z;eXnfB@P6jZIlK?Z9~5Xwh=%xd2oPx=cp~7LfHn|CaK=w5&X51hfjvAr8u_+II<1j zRDD29>3m|&zeBUPhwt{wT%{KBZPB0}bdEn)8DQHFQQYjd6J1F0j3<|VdhzbT8MEnV zp@SA-VA?dv0wCECMo)4gf7plSF3mPy(xMuGROu&DN8~UXZWP;no6bq4MMr37LYJyS zjdLj`A)#(%m~Ce$2m(|pESoTe5+#wmvMsJo#t-xw&c>UpjnimWfn}Mo}phb>d}8GA9_?kn7w;*WY<~=yY&4Cvpt( zyF@R0sC=Ni`r%*fSzGbzUCRWX)?kR`E#}(PZ`W>nAZ-f=-t6>1+)L}vm_nqHM5*J0 zA@t*g$n@&%&=2B@iHf4n?%adwW8_RL?00k4?+?55v`pwG8MGCD?ryzqaUJadhko|A z4)o*--#(hK`7xG=MzXgp!hXPB$YmAw9wC(*S0<M z6YCIOn|eO;&rKZE z$fYW;W}+63&}sM;A{`h&&-{6ZbLjj(ga4BJhB08Y^HT5&2~kS~5CpJvp94*3j;s@- zXJFL!qqDJdQ;2%5Ae2@Qd)-?E_hAG_En2hQ_*BcUK4q z?tk(p5Q|Y)jpu98QyEC{*)~*N$9#jn`robC_usWQ{=K~&k9YCp;ViLQj}tRD=kGS? zbzka`=QMHyQI0pucjxPxg#48H3>;onEDAtOp?02&G84%mKnIas%XT}hW*~VMO zfXNDH2+)oZ=GSn|r}uhnJ+}mT{nl9{H8^Nf`?T7UKR=87muEX#^gg3_hSqFsiAET} z?=t{(`Kfx;0SG;wx_(sy!xCjYMsDtDmS*WoH0R?Ah1fL9AKsb28S~U|PNa!ta1*uI z4hDQl<>7Z=WxI5u!JMB7N92zq!+! zP#r~W1s8vm;`lz0z0=XqfSvTIxY*6bWure-ia=XKp+N7w3dN<3&2yH?@N@=tS{vmY zac8&svN`u?dTU*r&98l{Mn)CPwCco238CclkpdN2@lD2$1kO`a`3?amYU=02IsqN66{!5ybhA&I`pxds2Kt)x-k6}CNGR>0gqWMMNS7Ec_tdNaHjdgnB zzuYZv08-gH2DC6wqzk_*=%KEp^$6M-uGVz^(-$o4x0@scc+#%C&RW%!<0%8fMz^Vl z=~rk%*;>n(B6p%Oevt)q2D_=+8>Tx*Ji-D3S*_$Z5vd7c|MmztuH50Sk+(fN(2`TL zKDE%Zs!4a|!OgSw?G@#mMcTf17ze~9z<-)O_S3}-p`_K-g@y_z49}t4;GuI?_={PQ zMmR?j{SO$6Zqq_$tL1CNCC{r!_^KAM^=Z;bB8olS9gwAtS^S7$#%xH~UE1OV-_t+k zVZhcuXU5mqwt$UtY@3l29j(Ewux^D^qLciVEn9@xYe3X)g{0;MseMU{NP!ChDoO66 z-1@ON+dJ9=#m=VFwp#9q^p%qG5-_ExSFH0lD@Ys~U6H+iTuA9(&vGrRjRWiHhoFk{ zDOK7zf<}X7t{i!j091_sAX)Dto~LJL&3gj`G_B`NhxvJVc|}B6+=)XIZFSNzu}aJg zb?a*PwlQhAvH31izvYIJ0IdX=!z3L6yHpeCR%N6DQW#NYgI7zA zX%&qF$wD7~yWbxDBsTXeZEn8WO_5@B=6mGN^!sD{qdRotxgPO8{jVoR>XYG=v`d;J zIO6X*P+Zk1{5kY6^TE2Uo%pQAc)onO3GuJCeqPl6X~Um#RFDk$G}eq3>O_Iu9fybDd2aV#oMtoK9&Q0oFbkf6CC5~1s)WbVco{NY zPhE}QL7u()fUS|%$Zqiz0~$auyd7$4N|XgH99v@XUGpHWv=Xdo)_=8YK#RQ>?2ny) z@je&CdY!BR)f!3i!O@YX`j>o$B|XzBZG!xa=e$$7xgFFgmEFz1&$_FTjZ?I9g&A$4 zKe0NTTiksM?tHJIW3f8Xn$5Z^*Lv`XBY?TSv;+H=u2i*jHcbEx&{z#sZPlt`PDNxj z2-6!Ny_REH=ik$PrPJV;G6Yogf7;H_yQe3Nm?=9@@Q~!Brm4DuMj})o{}D#AtHr)= zS>f=9)$(0UHIZbr5Pm{+nTTim#~`fHinyV$O_y)Yui_o{PRFLdp-9ul1<>}NuBW(w zpEx|bqBieK%%XM6)H`EF%vSLx=iq{CS)Y@&;9uTSw$BVEc-V(GfV zOvi_n7MHcJ0?y0s4YWqshlkZG4QknU&hd02?bWk*ir1NhbXLd_%bch&Ri2AjuU&!n zu1A%+y1Gr7C#R>%@@4Adn4KY!_mH8<-Z2p_BkF5OhNeY~Pi-RQ7>Fx&KEZ%VlLmyN zp{`VABMni~n+hfrC(KN?Q9P%V1U(yvREO}TYmpclC_?uicf^%iDlkQ-t^utDz-J_6 ztRjLl>54W}`7EEi4*G6;eu~vAhKC<8Qy`;cPKfnBwPv#}xj<>ub+T=B$C!_nxI!PN z0ELn|=n@{kQ0L?Bfg*1L3E4i#;P2*fuWgVme)nE)ehp?$++q z`IVm=G&GE)!Z7uoapWLxF`d-S?zp+9f{5>WbkIrZ{8$yk$woqXctAM^F!3GK0rS+0 zklm(=(dDUiwT24OcMyCyA0ZQ-1Xph1YIE3V-b;pM^;Q1-9EpWqQke}r%zSSC?Q3o# za=DvDz%hf?-uO63Hy~b{RW;lj4ULnlSO8uQursCH|2cstpV*T?)^c~3ojKWlTv>25 zh<3Kk3UVYmjk~T@%f_(uo!OJdmmJ!ZqaPDXnyqJ+cy98@qGVPm^XlA~*sONH;#dJl+5)mtOBvqH(>lO!f#%+-eL`ED~!Q z8ykSG`TE}fFLQIg9``=tHcEB50d44_F6u6IJ_eie;d$)q%j-0{^jzf z=*sEus)6PoxAavp(<31X39Mx_RpG@^JjkAYI^$ZB5v(*ynrOR@ccGmGsr!1NJM)3a zHQL;P_r2N{Pg5?M@IV`cN4n07<+#(<=~1;un##KqhA;N&J7;m-t-5A zxCCOZzUxW0#=a*fd;4l_z}QuX8Mc2hB;>Nqb5zUzlBgot_dm^-xe~AiwCX=+dL{|$ zIZSMvQ`&KD6gbQtdE%&%{KjxX#YQXURbg4kI8@HYjLLYsU%)$VAOUru2_mzhPD3MZ zh1rJdUWXF9gk4KCT*SgJGGx>l-QLkko&NE5ou5V z{CoWq`g&^dJ=-vcC}89$vJ{g*E$bz0OCp*A~Vzl4~)J zCw(faikYfhUGA8i=jHFT4;ju_=hBJ!;y5zkovL7%L(IL5<&hipB-Bu2D3t2-d9}3K zcwy(9ycvmMV$nt)M`a9%7WLovUIYW-CK9P!&4>MbuQb*O{GYcKs}*R8LKWy#LBPB_ zx-%SK7F}Fg+IG=Gpn1=e`R8%g7wCa-8tS$b76-bYwQIXZt$|&KCT**uvY|GF&evRl zh)o$x8rCv~yr7*~0=gL_+OA$%7RX|)%Rbi!=usAl?!_*ThLn`l4~cK>QUe0W8|ZmW z6bp8x!i`2$q(_pj-mrV4Qtkxra2>$NT3=kulw#5(Oqfmt0%Z|RaG{rx)c=e8kta$w z9|o2D4p*z9PiCOR)*jFHTacOQ$1s{V=ZxM$U?!_CwC;9RJn1CRM*t}egI=I zwXnd+$aTxMy%5!F=~ma^mB$Xo4)z0f3uR?x6Gi9HdIVgCac&FDpq_3%69QhJ_lWm< z?tC4*jvmEpMAq9!Od=q*Fx-t&WuoXR8-$)AEmZu@RE)o?P)o&ca#6M4>Qp!JtsT$b z(I|?_)1Z)(lhf7WI~l-+e@cUQOx$|C52i_%RT4!N8YXBFGmOa?)lbpcJqFSB#LCWr z{zk%On6;i2cF|nHaBdq95f=ltN2l<`LBgireCjJS?A;cbJN0&Ya(l6}X{Hyqx2>ui zGtNE`&A@uf&5hBuBRn7mcs2fDhoK=l?a#nAA?yUCs!Dk#C!mj zZ@bM+FPO1oy|UxlZ-RPFNNR9?z#0wWRwNE16B8+`B`=~To4 zE*>}HX0ywh;CM3-?*RA#4-)1sGX5#R9)ZxdA7MvwMZYl@GQ_~naCcx}08m%EySwi$ z{%|#j>jbI)ifu}&`3@|DT9HE8d~|679IGnSK01y*Q|Y2RrrBH&$SZ{?rM*t|FrYx4 zgbN*;7JrI+PK-#$8u*t2NeuvI3B7&$J>ZV{ri*sU z2WeOR6kFZccm<*@)Z%_%R3LhSN~yYqffAc{2j^Imf6@2M=|y^;sq`(u0Y)&QJTBXV z)ek7R!>?id2H`sJ4J=aWJZ0Qx$H(w=a7>;HC3{ltKMasVu2eUx97!GWVo5%&tQaNc zKU;!fww2bBj;+IN`|GSukN60sYrdQa8~vO1vY)K6#Fv1zjxccl1TRAz4ZXWN&}Ve3 zHA*}366MaM^Fp>hpj)D0oKU}QQ=5l8?gLA+18y=LDm(CPDw7kGVm?48=)GHY4&pc@ z@h7jO-R-%WZEkLUdkYyLgk&ly`JOi$Gfeb+3*wVRw*sYwyuJ7vZIDy{5vMaJ`m_f# zdePabgS7VKnyw@1_K1Tcf>jD75A$s&&7YwaRC0BD`U@dn=`rv>n(UvD<32hfz+o#J z-_ui)KoG{Ib7j5g_0@C|*uw_b*Vh66fk$weoti`1NBG7EK=p46Qm+fe{~n7~KYU^_ zpQHb}oG;+)&Lclg;ced{CK~+s zLjsuJA|Cx1`+kaiKV(X?mGh)#d`58lZDd?p(`xJr<5sIN>BeI%F2r6Tdy{CQD{AkX zjNoIs2d}O0v$8kcj4aJBUBpSWx%V0YSKE?&2)=?{WHC{N%cMN*<||o0U{I&ZCpv+V z|1Ny^&LP9-^gOT1)%&oMAXnnCj6Z212)>N2M;;!M>-`loz0 zTE#90GmoWi0uhs#ppw}-#o?&p38sZFQ3I(>0?Cczw3e!vI*G5G9Gf_gE;Q~A(Zwg2 zpazzj6lH|~Q6B9Dqi!tRPlNLc(BS_h&zF)m+<3@pOH0GWy7hdgCs|)arN<>wII711 zl%0|ppZbBT=Mtf9F1=AbGvfn5svCQ*Re%N9yyjD@gS>CP7ZJEyzJLI!b|tD@WgVtJ zwC`iAgucx%qMDKP-PSB(+JAz5mW&{J|Fy8F5=_DSFu@T5_RKdtII8dL1jpNoYP^{$;LYiy)k(K+OCDzb% zxd(}Vfh&j#$pf)hE!zfn5=umoU~{Boi39*c{Ix-Xj1Nmw5h~+hT-L&K0}&Z=K$2qU z6}X>J<rSdl41Zr5zHcWt&WSes}$8s?QAHqFb7Nzg6C&|u%mdkEV-u71 z<64U$w`~nQSz3b~KF=lA{L{gpN+xBb5;4_0q&=M--OS0}%zwBE)h3^~#)uH)*^ugb zSnOW}X*$b4l-RV?F^wvocbl+?aI{Z;DPF%AT63yj!@$_4p!XLhB}dQ&`+$z-Rl%zI zAd(e-1=P>%wqoW@ID@=E*oEZDl($DCAd(Bo4iUDFKGdz4uQv9nVl+eYv)Ni@ukAWX z!c4*cadZxTb-!&K&*sV2$u?H4PFS{=v1PYq8_UKzxus>>T<#Z_wT$I@zI*vmw)y8gA^UsC__dK+eBis$+fN0Ts6{ zP3~s4Q+^KvYBxWNUGb?cy4oF6p7+#cHnvCGnw_LFbAjni$cVqKW`>1o8Yc4e(?hik0EtIyZW7KlCa!v zP3mpsv5X*i6t2C;3TT?p%R7cDlYKrM7vmD|@r+l72{I->K3ibin}v`H?jS-; zl5rIMOXUdm1T--iIU_Eb1U}Ps?)-Q&I7|}Mybg*QjC5eV4%S~8X@oR+RoRpDlo2IA zWXf0Gl&i5Ff+G``t|(*kCop9tMns(bWHm&04C%7<4<1r)#>xQip6mh9j+Lj!dyO@p z&~4P`jlQ$M z0FQW3i5h9Wj-Y*~LsnMcJ6n+a zEjKT(>t8bX(pZic9&D?uQ=qfWZux|}nPGGOFGsjPB3Fl?2GZ+i$NOi&ehJLuE}odm zBGiIO3kEY^|4`pZ(j2SB*GepfWToFdLoQbocdQ8=tEAoivK+Av7RTTrxxdE8wb*Lu z1TV2=zkVw?W~wqv&*#^an%D?TXrw%%5pYfuA)rT%A&(E;EcT zF3cWxP|O7I#^Rz+W}vvTmj>cYdbB83n=;T;J~sxBjeM*gD7c`eT$z|9r}48=eC9ET&)ps+PBfuyiq>R*rZmVk_9zaY$N|Om z%!^1){&Lp`)U;CkVIZ?t(%eeIHfg#SZVN;NBJ_L%1{EkKN7<%S$>FvGy3O_&psoNC zYEy0g)jRo)>O80EG*QY9k3VUn?%>m-EmGeO1cgmNyQp-4Pw#SO{IHmN)uJfKLibu7TMDGN!QAk(|B@YQ=bvP3Qc_pgz(~`5eX15QNF9$jo6Lx}l z3TzFMaQZQDc$Pw#b)?ht?hnI*IDpv(QxfbamsNh@E*1Q=b zyZBj&Skc%(nH+KKDJ9J9uI5fmUJI?41?!Jptsy=4CPwwWjgY+yXN@Tf&WI3rR8O29lrWK?hw`|*oAPtr=@_G zrMuq&RDR=CHDzDH`ox%*sIu_3d6}6HgPi`C0N#b%qBpEtexFvLTq7|3z2^nY&Hsl? zVOIoBe5_73B&fW)ZaN_<0jRbTd8EKTogvv=9g1yuv+p(bSbZ7wq1mzIKZc~P+u&(l z85+E^D@zm~9eRXNPk8;7zg2&t8PtW(z63oD%8iE^5k#l+z=bBEX(&;%T2`(U?Qt5Q z6zTwpw}6Me^dsrSBV@5x)>f6ry zR|{)v6vo!@IA$|++Yk>X#9!i1C%WB_+J^<+Z{=1qGWyg=W}I9j-x{sjen?lT@vASV zMtH`?qepP)H@AY1|EB5+jLA1x+uK)mb{d)722O)aE*Fpcj0>G)lkkruJ#Y`3xNcq{ z)H8K4pT$0{a3C(2TN6bunM6yuMm3uS2SPq<{V5|by#REC*!Xy<1B90ub@{ohAIz1K z&x5(Ziy^_4Q6(qv9?l&w=MXBAqFKSK7Z(@Bis|`{{ujN-Epe@%fg}r6>K>^CgX#06 zlfN4>3+4Fzh->@fGSOE;jQPj;xo^}?Vpg=18$gW-tev{E=GKr zm*;=qtOk=Y2My9bpsww<+dj^voyOT(n##2I1m4))8-)BsiirN{3aV7c2ZAxwGXng zT?vGl1D{2bg2$3Mg%z!&gq|H&n@&0BAjNVc<~D@4;o4y1*af z#@}=Xrr2ppnl;l`$x2CnD>#!!drNV(BJw3;G6Jl#^}Y05nEgn~|JuKDHcZ#}o!b3F zw5-2gxJTMIjC7wlA81LsvBcrx*b)=XtTY!Tqt6w4(j=yvuXeO~$efG`4{do41dT8D3LkqvR?O%r8-TFZYFkotWM{Ja{TA zm7hh4+yOvFyZ~JvYMAcV+nSw>bHl%f`W1R}AW;%p;yb4S18%dtvyH;10Y!_wv=)YY z_fba0AHf=Bi4f*s0Ys)TT2d#)wWT&{_s<_7ok!Z!c-l&&#J6fe&NSoPl8CXXq28!Z zKMM{5pN5^f|}wx$l^!J7h2`-q2DlA!gqwOEe5aDT#GIjU+}#`%yB7fnD_ zvaz9os|Zz!&I>2K{Z>XsR7yv$;Ic}s{&Ctz+;xG6e{lA?OJD1U%=Yw0g|fm28Rw+Q zrNRQ75a3k>_$&2UDMEv*T01baLbUVV(1k&s(CSg+^|OykxD(Tdr3Y!ly48jcHE)lc zdf|XcfU`*^VDEXc32V^XX-=rc6L8r(g1(M5p!a;% z9Q!m2hsYR&X{?R02v0_(ZI~zdq$*JH`$O%fgO=qYBLnKr+Tcs<8U_x4da~c>(B~w* zqUG99Q3PfE{02RzVIMQDzgl@_olDOjVZf_<;rC+%gyWj-R}p-56p59E1pipattJxV zT+QLnWK=c1kvDgDQuY5-j;`A(CrM7kpXmB5`Fm%v1qI9lijBCB7?M*yqj6(A3Q-q2 zHi^pUlH8nh6Pl>KWK1W(|KVn1l4?0v^YZxnVr|Uo$f&fH;QUZ1y7HevMB?X1beE`QO2@mQYd*lq@Eb5~fm{d-OWoh!mfw67 z6qn1tU9U{Q%5-B{Qsj^K7(o(YlKrMoRSbdq*zbB$(B)dN>oTrC0^7}mFItlYU4-#% z);%!+SvP3Jfj`qe z3x}!|1WFx>ZVRBm4oTx?luA{7_|MM$P>~cxrLh&xt<||q*|#iz`aLuZVqy6MT_2sn zhOTu4M;cYJy|a_zi_wPFFjyqJ}l_9n0l3V1VfNdTvQCgq#*8|eJua`o_BgfEMHG1cqQI2T`^*ry2 z)g^>g2nchM7Hz6%g{Is_?o-Uj8BIrvThuuxfLGCEgIm+rxic)s-JoGzcN;$MA7|qK z6|1;#1X5r?qodkk)2IJhz!N9R<3LBZ6}M&+dHB%Hx*oXMT0AdlsWB~*D!3bdg$Mck zBmzzzS&uuQC5?f)DIHln%4zjIkkeU8T+v7nwv*X^)+hUEW<)Hj&DFfOZBj$h$|I74 z62~<``p`yV$VW|-wvzilc6Z8>s8X5x1*Qcw^L|R`etxz}94g-=AL9N2C5QKyS zp@NTt>OWjEx`he9=pDj0pWOr|B+^&o>Vra~6p(`jCEmfc55oK@E-=Q9Mp8v_ec&AJ z`0;l#OH1kt{D12xHN|Xz=n52>%hY~7?lb!tjeEj`#_BDu`RArZ8 z?NdTb#yv#C!N_PGxJ>^Ee1N2WKK%Zbn<+c_D#;8s4Sd#HPZjWd*?!)&IzJz(x|wdq z7q)VeYE-9IzIa@g$3EYjDmvn2cs1p^u-P(TPS^)nB>D$p)Rva=1DSX%E4d+p&UPBs z(sc!Wt5cqOu|(e&F*ou+pJ)R z`QNAo@V}0K1zCC%3+9J9QA_``a!;KCo;EstaGG*<6JBITlP*Ez*qE4|FMn5omaCyU zjiY>L-IvkFP*f*+*L9xeO8AG58pRWC3!(JV`e`w`6a?wfU(E%WP?c*<7E%<%Xt7)# zin;#Py#%=NaI9Sj=67LAeP)^VBq z(~$0mNhi2y7!V#wbiaO5T~s;&)1LTk3(0}m33LI78m4m?nF|RGRvP~f4apXCqI~b) zQCf<8rM8NtXn0l{M!lO5#j(>Ue&ZrP$iso5EK!|S5hLqcdg5^INK082BJMISz{Ct< zza_GST?qyQ6`lP@t@V|G^!H=va!cT!(UT@UtG_Z&?s&D;>y8zj`D~l2JlCD(z=G- z{(hF0p*@A=4CQEH`_x_ev^)suqq(kdA*Xw*7g|8z4^ zbffVeYerv7$rcCVWS-`!E=doTd)Ls}X|OKB$TGrxM(*!jX~eB5xiRYF@?1U zn;#ZG3`OY__fY#26Y31_i5Z1>mL-Xy3lgNVvC(61EVaO4?8TVxxOORKPq)foVR5m^ z`_?wu%P}?!nbA=4r3_@Le(K$Sj7JojoSYo1nXZUnCjVJFFWvz8QZdbD=P{zv3{z6Q z!JCkLd`h#;Q(|P`1d+gwF7TThqSJZ8c;fjMDL@=xO0T-!an z6RKM|jE`fZDWPkX*jL0l%gz`H-4^|IqpPH^#M3xhA?`XO5T!APDf8%ggvk&5EmDm_ z7DWU}1#qBPJ02O?Bxv?!-u;Kv{OTnGipJRg!07A<0pw(4g_;1;A7HABD5*@h`*77hg?a0)` zmUYdpnqN!t6zUMTt=4(>lo5QEw=MejvZJu_wJ%u2YlHvM&h0unUbr*5WO!!6#M?2$ zKNqW^bJX?aJ6iB`JGKRoFMHu7MOt{_7vHrr3s&N#lodlcdkiRkWncr*LMqXo&PwEGA(`lesK54L&yAzpHHaG!!Cz=Q&d0+%`7 z@5Mcr$9&E#5fo1>%e_P)M`1$@eZ=!@0)r@+ew=>|138x~AjHl@)@qg}BUKG5uKD)u8*t^46A~GWjqqz$W~-A2(E)|#7!iZ^B+Fzz zdBX3Q_kK4sA0}QI=xAB9lg(x3wNpUf_nE;&<+9tnAO9l@+xALHnnbNN@YWV$C*na- zMhRt5L~o~Etxv)~nIhRI;Z5`@!;8hkLGW2KV8)XS9uM7C4U=X~N9makPDnI$Om;0V zHmnc%1$=l8|NVr$HGA*267C+=os>!BcbokEd3UYVt&*map>v9#`3t9;2O_770-8A$ ziZ38D3c-$0Z*Z2#K=Kc%g3sW)e#!~j=OtN^fs(T~iK0<4Ai5oTA=Yqx-R9M1qs<9v z>2D6Sci{Mzz0}J#S!M7lX$I|ry1W;y3MuAbPGPDguEw-jPqtZ}03V;{(~{WH%b&eS z*SgKhfEim#Jrh2;{4had7Z#v?1di66)y|K^OR^5Z57A+N>sG1}lm43}BxS~yw5RdlLxFf>MUrd$5Lg~Z zt4$ZS*1;bmuuN1kJnopm-5ZlhX+{p4X*&nU6{sgL0^?wCLJ%8-qJ@4B^m&A{h$PMb zw!$P=%!g3h+`BQ%K3jTbK#lt+yD4e_&Ef?Ll14L$!aAglegZBZ4*hWTrV<@I; za@K$@84rO5A^t9xTVAeMCR&b{9o55ucC1Kg&SFzim1|U|+Aym2b6ZNnUu*K#1AZj7 z8w2=qlNj}bABfh2YC(Xk0^4WsQ4(4Qgr4X=d<#Si;wZ6)qa0!DEgcbOzHx?k7UJc- zCJMMD`bQP%2724cVe~8^Tn5NT5$O_3o{a3kOq4L`c*`}TIgE+z0y4*;4jnXSM${fQ zqgj0)Uk4|94tUJ~prpVeRT(QV%(C#VDe188uyl>^h@)EBsb_KQ>``3y5ON%5jQesb>Od z1wY_Vjz|Du-cwivE8_iVRH%DD(hqR;g1t@wJjq7`0mn6zsWDN5hCAw)XmoQp}O!+{n{jRab50enL*%&UO?h{+26c48;Tp;YHY15_BuxoNl zOHb==8P66$rk1EjXZj5YITZa8K;0#3 zJmEVpMlp1wRohStk}QiyAGAZEcPBIF+9l=3#WB~)|7MFr7+n~Bt*hG>)y8B3CywbV zxp&jWjjI}qD#hnITSs}swhyZ%E(aC!`5Rf%m2#C3ih}abVLyymG70bl?pcA;O|O9bv|T?Z78(SHi#iFT|THyNh_C}9)~@weD(JDI3a|*Q)Nx2o?gzgHqb|7q^UbIHI+ZL+3RGEwW`SR7VYsRk2V_v{ zUM4>PTJw!k3@tU(PkWFdLWFA+#9eqzgVY+9r(_y#X*#03+10U2a_++>pRL(FhipKx zXG^2v0eZLD1l$5p-9Zrw6f>}(x}BW_oiGgq8tJ?{~iSkMU$U)Z$Hbhzvk zUy-mvs!EYC&}t9g3-+cNh>l499055gMyILz>=9^yi!T3V;NJF3Ug__!cV|CuNX#h5 zsJAD2>|;g0`@teFus*@@B4to^Js!e^?*fDHx?u7`AdVKcCe-13dFpzL%iMsNuZ4|` zEhX(d+lPrd`}f<|{PppcgQA=3C$o^T(pu|;ycpsdTzt-cvxJ+LE)j1`+@Hz zLB=i4uzmw!2d>xVI_xRhBb-pe*Y)2oMnL}!0Q6gEMUe(2Qz8JRB{$eXw1W1u_(vKsXp6q&o#A}89=%Xfd%R!)SqumsG7T~B34I%$2iI3%*4Ly&wki7zD=G(>uPZDJ$&74S zjLOm*xtP;QUQ)v-FX^MOf{{Q5#E@5jX#HYKplQf?|$9` zfPa8TNpdP*P16>~JoT-Ais&tFVpDO^jhm=9@6ull{wFd5fY2^p| zNe>_)VR!Rksn-Ig71U9lFCfAWVJKBK9jUVWY-ca27Dn~aStgAsb&B{E0D{K1$SRM% zpmKjosWKq)C}m=Sr(wk;2OuD(7^W01?I=TG(`Z+=bPV|yp^paDJa+l=M$$QKvZ|6- zIqw6ygxMi`Rx)&2Y7HY+KllSN7nw!KUts7TfVdo>yTHe9dK(r*+GdB5tP`--kp~@$ zb){uU^Gc!t-5hVgYBECrP~+oih1yOfayF>naNVJ`T(E$I6^`|xojSQST#fvbd{?W7 zrZ^m)3C54+tA}hGa{j!N_~D%3Ay_5H1f5-NdWn(cJ|M|rs~i7ckd^kzYW;jZa9#}! z51X|bFc-&#i0I<73$cRMHzwN@5QXsHT?z1#tQe!_AC`Cpjz+W!Lhz$&pAGd6iV*#Z+~5^~S~qfIlQGp|&o_ z2caw!c%*#~ISH)we0h1H0yuL3Ev6w@r@~soSOk@#<<7Il;I=#s3JpsLRT`OZ8*nJ> zhi*!5Xu^g5T!AL`#S#K{rj~R;26<|uWI-A?3viVMGUfMY;kV*!l+ZG(AVHmel9xVq zu&pEjfX~01JU|E?KOB%zKHY><&X8`Yk%0^vWf9fzFvwWk?k~2WP#SB;L-L{a*`)%+ zT=n)3b*7s$OkJ=Oh+_ioMmOx?8U<2_e;ciH4=GsdCfPkHLvPO7&x5O+3$wwux{7OF zzC)Y`Ug%}U$n8E-t;(!IGDRuGjtn2W@S}B-HQ+)`3y2j*VH^m8)C$r*rD`8y6&t{T zVS?Og3F;7Lb?#b`hf2Q@<`lyOcS6#pg8&A)l%d3jV3KgvLb%O$E7{WTX zf<=3PQn;x$jUOn1bGf^_>uX^8VP?jh2Qz_{F@M^8q-9GV(7iC8fV=hR-Y;{uOnolN zelT|NRGn)8=LCi(orrOHo98SC+I;3}a`^HQ6m%v}bR6|dlC(U^qx+py3#53qfLJ=} zPVY$U56Kd(xtgo-2-=K{qpS4tq*@)|wyS}xBby2wy*f-FpPIw8DQqN~am0sn`OMNG zFCn!ESHk1AP&GPF4#%7g zLQ|-r5P~x{0n~n5))|pD6aw*$cswD|ps6`4UEJwq%=U zMA;fR&o&ttCU7QtDXxIk9s;G@N%#aicfPmVnqA({=YAsp>LppFDL>mnqE2z`ti!yr zN`oQCH!CAR^Q)A^KClKb(w_buGeCwiV_t2SE4SH{WU2r-p>--m^O**7M*UX|SPtA8# zPCh2)ogKl7F>JF3_>M}fzR&lUf*YRjEDdZcJkhS;-{p>l``M(SI*mpqfvXz20$B>e zM`(@j#?cxb6Q)gFO9+ng@ba0S)9^zWD7onnNl8=4YsF)a zGdNIoM(;*vmUTXj*?Xe$k{8y|c2rh)Q<|LOkjtp-h`#)7F*m-vydGeT8&LovHHGJrfvuSFaOsFYAs51*eI9Tp_A+ivA^lQ*yqKr^C#Ss(efn zC}&l`Ct`Ck{=h0Ioc?Lc%fJy6RgOvk+vq=~DVw^+-2z*m_61BaH_k9|hdKhO?zflo z;o;%m{!b1gZ@}aN=O4B7G&Ekf(`|`sM`(fUVD9XE0fg%V0;QUH$31Gb45@Zvo~tW8 zJ9!Ec3F{WHQmB5~ZS#h*wSvt6KLdEUvJFq5m9lB*M1Di|lp@{$mM|>WvFm21uFHLG z_{v6?Ua5LLDe>-b{{XNOfn3@9$uaZr(|R57J>`plsOa<5yYn1TucYymTZaeS7UKW3 zF0~yU*l`r(NU*6P)TmO-?Czt8km}%O9=MO*S%QJ@-s7`R<|#10TwxT=Q}d1O4Q{iP zxr)2X6aAS?h{(?dJOEw5F5|O@5WYzjBa4H>UQur_!m&|8iKPWy2|m&cLyC^Sl-q@9 zHP>d{AElc0y7>9M0G@~2t=?2T?3^cas*wXu_&|O|ZusMfLuzHqmg8zH5%*vPvL)Ls zNku!I7l+|^g1(6G$1n?LN2X#W7MchAig*6;b~Nfdlk>m)|MHd*6qd)f1M(tenJAg~IA zPVd$hc_zoV8nhZj`O?;S2eWLG*okQS_rX`dsQGW$!+!sq@DB6ONIWCG#N-e4kNWA#( zCDZs@wq?6p6ea7ZuQF-RV6G)Kb}PREP0V$jh{%mA9zXuASkf1nt}Cr9`di?~On(?P zv#aWyQ4d_suO_d1CLD$>J+DPS!o_!=5zfnL;pm#R(L=95rwB7l30{x}+-|$Y%H>wK z&r|8?xZOURJ+Kmqzp1wyDcNqB&?C-H5+=knSX^naEobKE@uidH53=ZCtNjO7&o4KY zN-SuML83 z?ez|TB9BkaUlMpbD@#aSez5>?oMC*>=s0X;(NtL3cYR4rtFZ(+U20_4EtYz&%IZHi zJAsV#=Mh?*5qyR)J~LD?hyfYjgSEQ%aCAk|R_Whys|a5=fWW^5jf(zNU3=1*v>4`* zy+J=_DOuHlkc*}qo_Y^--V__6)8S&H;{=vJxyF#cTCvGrUR_HgPW~YXNfLPl@O=`S z*uV~^o#A-9Jvr}w9wsGmT4t~K`7u8S?MD+{+&HPKbl1P%K2sxIPs}JX!-p~fC9Qc3 z$rSmv7!zv%&;@-vSY11g`)6!83kCxe4zih2|VOtvP zpyz)@vJ;a(ly;jHjuC9SiH8Z&4rUJXeKIpxJ@mhFgs%&!UJeDjAL6Wb$&krh5bG*Q z+xm?P^Oowb)AZ%Yl->IQ*X+5ijF~7DaC+$I=y1Z#!x?NMC6bEs*)7pUh@u5wepF7f z)^#>MpnjU!r62JoG0M#wAoJOxD8*4(%YL}s+S(HQ?TdGH@yF!t5eUAnk*KqieINVs z_s7s7=SS4Rx)B_7HZx8Mi-4EgB^uP;diM-D4f1~C88rnKPI^1meR%?xLy>otsNOR& znGaZ526lFK{zmgD=n&uh)>bc9*Q6Pj6i}VGL6MFmeX}7L1+S5VL0;3kU!XsmCVBaj zqU7u-zEXyA8#M;K=@KE`$?U(w-Xs8U5c6{sw~dTGcJ3H79-5NGI1?83Sr=Ez0SWkz z)Gx+9r;T%^s)Y~0V?IsE%P}BaWO;YvjQ90NPW&!g8lf$L7^_@6DjzC0@#xVmyXwpJ z`!_dzY#1jPnot_tkq4?}kWsq`?QBVK7mob6gNBafJI`_dbgu?AyVRd92}fPMWn#aT ze^CC*aU`pX2f3gue6OE1SGE0?V2PeW`1{$4?Pg4GLgh+l8z%ZO;O#Cz3cV8udcS<< z;dJU`neR+9Hz9$O`l_?|wkZ9TW}Kz0m?~(vGmW?+4vx7ID0cy>N(#X-z(D}kSq%f; z=fMGqeqSoHMB#|oa^*OgNm{pD2a}O_Qg09~t}V?a=XKgu$~z##553Dc<3xILvyll1 zcrS$87VK8aaQ#P5>b8t-nkj6Jshml&1JEb`{l=gaa!4++`kq!xpP5xD_>a~ov?LR#Q*idN5V7b5nM+%xu;N`~ zbS|xosG`U`TL4U%AAEuwxqot1qDO?ZZ{Fg)*F)JYlK3Er9*SA?0ATw|P{6$Rz8oY1 zR)yT~?P#)mI4m><+opGD>(u670~DA`r?gLh??p56G*oHQMR6Qn8SQG8mct2cXT#T`?(}33qOCi#l}v7_0$@di2q2hNY59|&^^#Z) zvQChb2=^3%_W1A1zg;qkkTspPH`V$`t8Q~C=<~$lDz6~9hpeAf{gbC7k~SQkxIZD( zYmr007=!~^*=X~yb9bM&e2SCPFv3D20DaBVV)eB0Q{SK)(NCmbuz=$shXJoCC6km4 zMsP&ek_iyFP+G&Pyp}r9Km~@FR7>_Qw)(loGD#z@=T)u9euuAAFxI74T60#9Ue$>e zU!iB4Q$=}pIQs%nsej-3C{DJkb^U_^J+(Au;5|^D0r(H3B>@WPUeGB@_PN6|g*8!%QdG#(u(E6yYn@Rq z_yLj40=@PsDU7_m2XF`Fj+?;CIpP7(3`zRzyu5Hz1T1Kr1l#*m`irIs6$ks4-$tZL z26$hi42BvZdw#5zUXFV7hdDoCek-CQP`{2XQY^Y_wf^u#_(XUR*9|QZW3WrPDfr?N zJmk~!k>%UpIYu57EA3UH5Rjr2}G!6hrLJ$S|Tnxfwe~e zE8V^oRdJKHyf&4~|uxVK`)$6tjs(5jd-z<@WotnlPut=k^z1`;s43!U~X?bn5 zTMCpasX?aUp+|V(+a@i30H2m~)SE#6&Y8cypG9tqlC0{R3R{8H<8J(J5&TuwsCAQ@ z_wFl}!6P6i&IpT%iD6JT2n4G}7jnq-WEo6x*e06mqw%%U-^T3n(0_oh7-Z@O4EcXv z^lNK^qqP;g0?V>uykV_qfhxhdo2+WA^k@$Xg!kRMcR)+daQT1*0V>}a1}Fg`>Y?5V zFK*~Ws1~WtXw4Ziy#6n)E<;=$pKNW>im>@})zJ+Pb1fg#_b`EYP<0NzY@DIKq@*lU0LB3M+|cS8?$k5GN6#($z0^#VPn#Dg~_kq~|pRngA|!RTXUP;O0_2`poYnNCCzDoau-2qw(jLGrHPy+2Y&a0b$>Yi-pN?24VU2-} z4FGJ&)qH_-;_hTwsyGa0b7RN-!JxjakqbR}8g@#xztXjnW51t`A@4bz}4TMvXWG^9l-6jvw1TQp;tZE6JdNFeBuTh+Jpt+ z!ruZxpr0iYW5i(RG2pDWUu#L^Pv^Gi&eZBZ;Nt9ZtEBZeNS-VNMT>Y)prZ4TH-nr0=41;T8H?fXC%$5MtQ7J%SSU}UqxVZFevu;YCk&xLv1!O~Vn+*Q`q%y; zu=M)Sx~9U$ZA`FJs@8dLfJwmpMl3LxAhJx5vO#*Mze6lQNZM&KPvyt5E}Pztso`&9Dt|;v~xhySYBIGIz{ZRWaNJ!oxDPavs$RTug-{u`*H%j zxbl}Uu&@Ac5U8X95Vi9w7e+CiPxMN1; zX8a4{Cu0sbp_zOq0JI93AOd+J9%zK3e35_}3gPO@p>{L!{ho5dVGy_-A4K5-!05G5 zo2X*-tn&u)EJXrX6g9gcr)GHw=H_($Z6mlfv`bTSiinMTs--T8nW(i$t+uH9L$uu= z!)lSSxO2F`mPrY1On1cj4^vw=96UEFoBQYdON`Zi0$S+m5n=A*;$42W{3>PsiYNqMjtrF!1@{7%wPbTqJ=szvEM_I(G*h>pe9$c#%x;2nY^v4gM z@C333$3T~DkawB|aF2-XDf<$b_iU1mI(odd%ktj%VYjE?Ar68ce~rb~cb(qZNLGP2 zjqY+sY^#>XBnL85Ok6zLc$a7kWEMg#DV5RZ-(Mh_Vq1)sZve-Y>$zh%B7AZhtx=s@ z=%A^Pe>UO82UCoT5GJHuSy@?NFzgY`6ju~cW-MFE3TqfYPChm-Ip&
          u%;ww~Hk z0r5!7f@yG2&h#%ELOHZyXpY;g#tHSriER#fsdK?`=8`h4TlE*8#HyP6mfIo3&idau zyfoergU z^qga6^!bgtnpUHU2^}%#qmt&>z<@@W#A3C82dVv9S?iu9bds;zRkuTK<&+4#BV_l% zl{ToTiBPVAY5UbCOo74e3Uvxi1UB+m@*U*^e$8dKK}H8})(*8@s5;90cI_8&J`j_C zljYXA2s=U8IBy%YxkDBmfcB3O+Gj&8Xj7h_cdsEPml|(80iWmh!$`d;OQU> zwjk?=giHK(SC1BLaGjx7X=xL$=D?C4Fn{|g9!Og;Oh63!K&v_`i%$c&(a~W2Z=-~S zUS{x`W-E)_GwN$$N+En2q%xm}Inz8@)|c_hG|5Vv;8&@73Xwbt$jP%ql;s+a@Oc2m zy;03&LW29~`1mp4)JWb9Ul-(GoEefh0|(KtgGT_N(3sixAbq&sHaQES^R0Etr^Y~v zk&&ZrXduim3(kvuG&Hf?^U+=hts8aDe?2VQu=6t6%ku(COrK7rc*SL&V~K=q0#7q*r|G&KFcZjCPvyFQu+O#)V&u=yRbD7!~7%u%Reko zUQ1?(DCIC}11m7odbD7US1%>?8CKgIS~R*7zTd1kQbwRCZBafH=~qR?G5b7@vD1hS z;u-hS{OW?tdZ{#m;KaRRCd#yb-e6f+5e@+jeNF>Ap}}K{qy=#(_XY70pDl8uCOEsg zY7A@wKi;esMrDSrR94*q2H(3gisR|9&;p4+A34*-Yog0}!K7d3a&I~``^n{GwdZ!F z^5RCE9gw7tM(bYv_hZH?9W0_;R?j!(W=c;5zyIy69}a~rEs)+{%Sz6X%E&Xzzs})K zv^BkSLbQecmk~d+CT8Pb5Pmpjt`hg)i=OLn9+28yk6*7Q1d5xE@i@X17^g zVxKdOIi$Xd3?ib05At2xjmA3OYh^ZNf8RJl7fjz0_pq_~q&L&j&J!T$*)&NMf^ozC zpX@8Wr&?Q5yEB3=u3#*0I2}FHiz)Kh-|6Y;<6~Pl0`+`)Fn+N+z>qMgLK{}o-qhN{ zP&~4xzX~t%%4l?64e-D<81OKkr4EbD-1$?#vd6OE;V`wSt8%%R?pv#G!b~U*U;kXt zC*}#Yx~27z&tR^21O^EE!EcQ$PWJ$Wn-m<`N`ve`*^3sNB6`%a?&COfAJXq^;| z-QJaN`b~72HS_TYhkBd8$@LhDbNZGLzff-c?tES<`2rof@OE_qnHAQ=XwA69kS?zI zAZD{h`C6A>yNu6&eY-Con;oNEP`H^eLbLIEMR?S{BFA>HO;?&YUW@5{^kd_C)8&a* zaZGM&wNMLxA|IKBG3wXRryH}a-?)0Oud9EGb0|Y#X}YVd1Cqdq#*hvNcF-OIQ}7&` z48AoDDu6HOsH>wL{@L0ZH5gr3e1{!FM#Gj2Sr=YvNz(#bQ!z7=&~h4L7n?g&UzU@~ z?xj%Q*SO=uw_^>V+G~&!6Z0L}$mI_QrUPEX>&={M)oTA@S}nj1d)wTUz+eIUN|^OS z+lrQoZAdJGVS6Q-gj?CexYZ^JWXKKw@XAfnVHchlw2+_6gy?J&b?vE?H9meW!93{J zM;Vi#gIN}qt)_5wm+=c>vpMQ&Obg{mi+Fl^vK~MJjk!Q#d!#MxB=k(CKIW=BawZG( zFhT{aYI|AO|Kj9px#!XDLblc4a=ZO(v-q!e^qq41-Wl9|45L0wGCQ!J#m4m#l~Iq$ z2nHlQbaZs~_0F_pXBU^msAp3m^kzHxyjtUQVY`O1ia+QL25m#ZoVFMP{RDAa&~ELe zpv$nlSVyel4bUot=LK`TP@n9bNFXO6) zmv>ot-7D$mfA8$r*V9)R!@Kq54P>Q}lc0`m_-X-ShrMY0h+jWyF2C&MlKl9EB+m>Qk zlOVUZ{#L&^5!8|CWa4aC+wXokv_3|BeShaF+ za(*`-Sw5O(0HKwXmI5R%#-T0Xy9o~u-%OG;p&R(EMxP%RF5D^>g-2n7C8NsN(iM75 zfnxGdS^m5cQ2uZ=(HIEx%o1cDL;DEqG+FEPX?AE|7YCsMJGjIF*f-GS0J?vuFShcf zBm&mXE(5`n3`&JGORPXIQ?Tbw6W=v5@p|zf89Qjcpipp=6~8hL{((#%6lcl4D8tg);-}_RWW1EtAxVI zG3n{+m+MpGq=%#XBUc=7fpY3;!@S&~aRttk6BCgXp1__^IFds&Mrh2f#$f@lqc5y4 zFuViEXj#|oCbe;{U#rg5Rmhk}$>v`R`i^H}i6-OWl}je7Nr(Bn4*{>$0J5FFF!Vu6 zqf4;63yuj2gYCioeu;G{3c=XX)5TUu$6?a`vS%XqOV%;N328+a^Gxx`*gC)Pa!G)| zTwP)|q6~gs1_0cRAp}Sz214*g5eaJ4EI~PAfX{OZNC+YkLK?h7rkGj!D((esNkN@4 z`O0KB!4M+E7h(*Amx}^xJ)r0Ww4Pc4&J!covbL@0uB@!A6nzp)>cGFhU&B6Mn0AV7R3Kr(d481d z@sDTm>drmA@BKG54x}$!rm{?hxG2Numuf{Z41fE2fZK`xYsLUkLA@MPQ{8_N7gkZm z$6BS{O7C_Ze$nffJpSp0YXx<};=%JJO@FtW%psk*^#JX@teE6c_JrX&{xlHqax&R?Z z7nfRil0%V%(7?lDS4fxza76j|_)rrg=Etm>1qR6;LO}}Y27glKHv>{dPjK#KUP?Ah zSPSOga|aVLeD@8v3NodB<_4+;9PzFz-&|8_Y{1t`{6S}IH$ zo!mbN#dvm4`osF2Z0e3|G2V;{c8w#biT$0Sj>{7hpUgOreiVGblVj>>2gDD9Z!?(~ z?zu{RP-3B(f9m+u)VG{%Ak06;N+s@_pG8V9Rb*4n`?i{bLh`s!MaDAYueLQ`70ySD zF+X{1C3QA~F)4*8!)N~K(>^d?05?>fYqWTak~;@#5DRidJzp`y~pAP(&4kjs=En_D4kBZ9rh7KEn$t#riWef%~N%7HnLwv7DH z3_^qOXbk+ql75oIw&2iUg<{$};jO%!%XCf?N-C;%3rN{SqvbSt)*a(W0=)`MJFU!* zOp4F&cwu++P|8?D5+;lkOrm$6roO)q0G7`I`IE`1DMCLcOz1l~CY)1qD=WmW;#)Ke z>KVv6W5HhXYRbz}iQE|LZA13>e!lb?qr_PrD?|#d!*bMUbka9lw$U36ZEY-yGM1(@ z$n#8X;h}Jep#{I=|0C5($Ji?a`$Xt6 zNk|M14vN-+J_RBY_xJa4NvhSVa7^Bn>m>9O)6pff3iKdEfT?ggF6_7BE#afVMmSNZ zm5$#R)P&&q6ij16Hp_EiBFa@3wR2`>=IYg}OG`^#%0Sn>By_QAy12L~r(TTm=g*(N zdGki3oN!Vum{1NWH6jU7OD1o@r^@>-%Yq}XSAwsp8a8&R`p{hDSkd_;)HM*)Jhk#k z9#Y|iFhrCX<2ywFPE1TpPfsr_EXXpD>t6KvI6bS_~pp_NgBMt$%9ir$@DDH;f*gi$)a}FJ4?-T@@{KWMri1GtmTewdmjdxSC@8-qKSmX<^XE&3>%dd7Nk zJslhz{Q2jfPo6vx)l@hMgHV8x-GC+`w>1MvLh#e*qI%;C8`O6)f;wk@`g@e_Jz+@5 z(g!^t$wc9Gzj_G3$;n9(fWknLgh)^8CG;i<1wu2S;@aBU)2B~2H#e)*Duf|rpx{3- z&~=~U(b$KiC2d7Rzo4ro2aSBJoqWy@foA!rvPfvEAN-C?D1q07x%7YPVhG60p`oEl zr7|@&wYa#ru&{uUvb~3%B_VKpeEj_R^S}T8`^AeFZ{NNZbre(t4-vI5;@ihLck``h|pm$UtG^ zqeqXPJbALcy)E0qDld4QQL?zC(>Qa! zYqm|KLF=EvUi2ms2yK%!f%Jt68^_1TXJ= zQs>s7cLrZENj4_!#wl5+gB+S{K{K-H2istZnKAIVVd4#Z(a*is(vPtRoed)l2XLpTS&E*&ZDo75Y^wmHGMk%a<<;C;ORx|6GESgaF7UvhL~Ar;v)f zySo@yN|DG~VK9xHB=`8}tEQ&&VlWjhi-fgw|NpE*Xxs&Abj~aRsMD;YQ1f}+cTiK2 z5kI_$p1i}u!@|JX*;!$r&<_FnqAz<#pd_IMT%+qTL@J8iNF7xkkzP|vGHgImZ6fMN zM*NK=^j$cHgobiD;1P7L*7bAh+!vpC&TFznkBiGFdcenYtdGU-6o9OJ9~l`LA0MBd zo)-Fv3W|4&zRn$ll7wNP3<^5x#>R#SM$|*0b&c=}f>C==H4h;kU95KEb>Dg85|QXr zk)_6dP%#*I1PMtS^A(-Q0`QdqHS|h5U^EHII?Y26Nyj7*=wr%@yn-fZ6B84nd5YnO z_ynAyB{WJB<^ejYFzVT}XTrwq?QJm^4-O8Xl+t;l-KTWn=vbl-`UgpSZf;HlAR>W9U(C)$Ny3I8 z^blTwiaR?y`}_MsMKqYAhlXs_6~lh2O|XF2cvD?Ez-Uy^S}5*sNNB7+O+uf6QM!Aw zpZ2Lsjp%gIN!XZ%D^}mo(2zVr`4r8wKSK^DMM=VcAR1vQ>RXI1+i{vD3|%PTt|gvJlTfEt>$Fdj&?RZv zOgdwgtvm{^#Ogzr7q(-9KZb{|3no6tQ_%Hp%dd4;quy4ae@VEzyNia1APg3qc8&aV zf|AJ=rdz2~D@kOa(C_f@@Hhs_qbL}4F1QYFm^4%)2A!7?p2b6#cBmS+r%^jyBGaPK zz$1z~)BF>>LeI}LXU<^!7>LIngmk@fR*w#5a3?`{?)*SJ%uF`qQDkY+i=r6xDG5cB z6wWY3+j$iaWCcm6Get=ltSTL`&?(?-Mr^A1As)3D)S}X;qdDH1C_R9YM;A2tqKtI2 z0k7dFsPP8rS9mPkAM~+zF&jm?7QH~)ioPQux?NEJ)Fh0O zDNb7K=oo`>a=MKAdirUgml_7@gc~CX@z;3Ec>xTU&Z^;R*8A^5%BN0zaVqXNkTlrU zZcm*8%AQMl`U=R2zh?CyInQ}@x|lFZ7uxH~p><;hLPW;)l~nAnjG_1M-;3p@nMP@Y zb>Is_W3ZXXyGx5r`aeN1*gsg=##;j&*w}>Hm*YYtbQ#_nUj@}014(1gTDtALRz9^C z)X&KV>!=8jOE_=3pb2{(#dCPHbfw(ZHeuxDB;Y{`b!3j zJ3+TlP_%0AsG+Xl%$YNlN~P#E(r)w}2_eu(#nib_t`0-RV3i=?)5dQl%un)X^(%g) zUMrbK7sU$JkrEe5;PzR;PdZOh8SOxSjJet*p$l~rHK%dFtGb|+*1jCQ{}6y$N9`Z< z9SMQD_MR+-w{PEKkdvQf=Dane$!TO@F!)ZbOiNnIM&}u>TAUiLx_TsZ>HCY3U`ClV zI{847%NBX%g;`2(r2L$biWYwL)mIY}6Q47>d-!DTAM_~+8v)n7tRYlUxKK#IshVAv z7H&rFX*wCHtJ46m!4=HEfC8+4L@($_wHq{4^p#s*Sn1EZkl)qk9-Tc=+H=FV1_uYn z$H(>I_btuxQIfDB7~Ri#l91c7{!nS*Jh^8yOr!L25&M$RB_sr4g1I$R9l0m|9{M{I zd2xJ0R~{5TbN1}nv9Ynw^(KvH3FHz%lq75kPI^{lvF`5f3Uwnjq7-SF{)!h$?8VTW zi4qs?$jHd(=&1gdyk4zV<@fOcyePjczkiDTEc(>;AW9N;5X`m6U-4NNF6&z-`tcI& zL`lLDB??C0xr0lT=roihEK#Chlq4)sqF|IHEK#Ch{2yDykWdKcYZXhx;AWdO;ATlsAGaxZFIx;poF)$!2FflMNFqEPc00007bV*G` z2j>MA4IBuXb3l~<000SaNLh0L01FZT01FZU(%pXi00004XF*Lt006O%3;baP0007Q zP)t-s|NsB`)ynpnE`^rejLVKx8${{8aq z^qhd~UNHUd>HhZf{ORKS=HBpqS@^)M{qXDi-O=uHO7^dq|NQ#>?B)C3)a`9U_OqS- z`uOvcbnRj{`Np*T=;8LYp6+Ei^O14<Qf`{cvAGEi22jT`qIVmi)!|; zne(EE`N_HY%DVNclJusG?|@wMmU!}uYXAKF@{Miz(ZbwMBH&OWk24m%L>;t29NA4F zkTVv5; zulw1~{P_31hjxoXD~LZRw|#5#=Hlzv(zADDi9aZiNHM~Xe)Ob^@Z#O9ZCRmSN#@VT z-@>?W)H>g2_gg{x~<&!(FD-PFvXlfR35*0H7a zt(KHbHiSAOBqscBVrEEJ<*Ox3TW&ZU>1TSj~_ z8Ln_$&GqvN5B z%19sAbyn}gw7f$d-H~|iZ%62_oz;9`?O!s=WkkP8B<#AX`q#?ymwE4XP3>Sa>|HVK zYC-P5u=v{0^OAD;&A;7nOw~*v?XjKDa8m5Ntntaa)_-I4&%)S-YuAHm#8xuxy{`M? z+Vwy8m;e9(0d!JMQvg8b*k%9#17}G@K~#9!otEcclQ9s-!z%C;sMf+Sse+t1sbQT+6bg zn7QxTy7~rwDWdT$MU4^sYD0ZpEeDwSdr@UH8q?82bqxyi%UAdfc>M-_)>NbQbZA(` z-w~9jWkvxkuK?h|!@|l(kL}!(r-Gc^ZD^j07`I>CBb3=wcnaB_J|lu>&%t5g`3nFP;Gl*(4jahi0&pPj;Gz7(_NX2? zdhGZK_?^q(pq5JjJD6i}R39uk1Zd_`F2e>gOlH=8in22KRSwMHQjSaV)HECDTo6o4 z<&zwU<=4F=7mN+!MV5g(cJA7}XK!5czGNz>#9m_(+_H6BLgI#v+Y{ocpjvpWkU}M1 z#UpUjW^yu=x7FeAajXPfy=HCPy7iLDSo}rS3){!Ij!leUswYc8v@0%}ECKeS6B)Gv z(@JQ%3L8g7N{fzzGPewZ%V`T(+Y&q)kZ4*crF`)c2riu~npJ5ho0f|_+ zh**&{4`jPCYc>(gIoP&3d81?BbB6R7W`^7LlX3QzY}$0n2@0u3YstjePM$(N`>C>c z;zDibL Date: Wed, 26 Feb 2025 23:08:15 -0500 Subject: [PATCH 098/119] Fixes --- scenarios/AksOpenAiTerraform/README.md | 49 +++++++------------ scenarios/AksOpenAiTerraform/infra/main.tf | 21 ++++---- scenarios/AksOpenAiTerraform/infra/outputs.tf | 4 ++ .../AksOpenAiTerraform/magic8ball/app.py | 3 +- .../AksOpenAiTerraform/quickstart-app.yml | 4 +- 5 files changed, 37 insertions(+), 44 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index 8ea7e3fc1..33bfc52d7 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -11,50 +11,39 @@ ms.custom: innovation-engine, linux-related-content ## Provision Resources Run terraform to provision all the required Azure resources ```bash -# DELETE -export EMAIL="ariaamini@microsoft.com" -export SUBSCRIPTION_ID="b7684763-6bf2-4be5-8fdd-f9fadb0f27a1" +# Terraform parses TF_VAR_* (Ex: TF_VAR_xname -> xname) +export TF_VAR_location="westus3" +export TF_VAR_kubernetes_version="1.30.7" +export TF_VAR_model_name="gpt-4o-mini" +export TF_VAR_model_version="2024-07-18" -# Define input vars -export LOCATION="westus3" -export KUBERNETES_VERSION="1.30.7" -export AZURE_OPENAI_MODEL="gpt-4o-mini" -export AZURE_OPENAI_VERSION="2024-07-18" - -# Run Terraform -export TF_VAR_location=$LOCATION # $TF_VAR_example_name will be read as var example_name by terraform. -export TF_VAR_kubernetes_version=$KUBERNETES_VERSION -export TF_VAR_model_name=$AZURE_OPENAI_MODEL -export TF_VAR_model_version=$AZURE_OPENAI_VERSION -export ARM_SUBSCRIPTION_ID=$SUBSCRIPTION_ID # Used by terraform to find sub. terraform -chdir=infra init -terraform -chdir=infra apply - -# Save outputs -export RESOURCE_GROUP=$(terraform -chdir=infra output -raw resource_group_name) -export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform -chdir=infra output -raw workload_identity_client_id) -export AZURE_OPENAI_ENDPOINT=$(terraform -chdir=infra output -raw openai_endpoint) -export ACR_LOGIN_URL=$(terraform -chdir=infra output -raw acr_login_url) -export IMAGE="$ACR_LOGIN_URL/magic8ball:v1" +terraform -chdir=infra apply -auto-approve ``` -# Login to AKS +## Login to Cluster ```bash +RESOURCE_GROUP=$(terraform -chdir=infra output -raw resource_group_name) az aks get-credentials --admin --name AksCluster --resource-group $RESOURCE_GROUP --subscription $SUBSCRIPTION_ID ``` -## Build Dockerfile +## Deploy ```bash +## Build Dockerfile +ACR_LOGIN_URL=$(terraform -chdir=infra output -raw acr_login_url) +IMAGE="$ACR_LOGIN_URL/magic8ball:v1" az acr login --name $ACR_LOGIN_URL docker build -t $IMAGE ./magic8ball --push -``` -# Deploy App -```bash -envsubst < quickstart-app.yml | kubectl apply -f - +# Apply Manifest File +export IMAGE +export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform -chdir=infra output -raw workload_identity_client_id) +export AZURE_OPENAI_DEPLOYMENT=$(terraform -chdir=infra output -raw openai_deployment) +export AZURE_OPENAI_ENDPOINT=$(terraform -chdir=infra output -raw openai_endpoint) +envsubst < quickstart-app.yml | kubectl apply -f -``` ``` -# Wait for public IP +## Wait for public IP ```bash kubectl wait --for=jsonpath="{.status.loadBalancer.ingress[0].ip}" service/magic8ball-service PUBLIC_IP=$(kubectl get service/magic8ball-service -o=jsonpath="{.status.loadBalancer.ingress[0].ip}") diff --git a/scenarios/AksOpenAiTerraform/infra/main.tf b/scenarios/AksOpenAiTerraform/infra/main.tf index 4a9d39708..1eb39783a 100644 --- a/scenarios/AksOpenAiTerraform/infra/main.tf +++ b/scenarios/AksOpenAiTerraform/infra/main.tf @@ -79,12 +79,11 @@ resource "azurerm_user_assigned_identity" "workload" { } resource "azurerm_federated_identity_credential" "this" { - name = "FederatedIdentity" - resource_group_name = azurerm_resource_group.main.name - + name = azurerm_user_assigned_identity.workload.name + resource_group_name = azurerm_user_assigned_identity.workload.resource_group_name + parent_id = azurerm_user_assigned_identity.workload.id audience = ["api://AzureADTokenExchange"] issuer = azurerm_kubernetes_cluster.main.oidc_issuer_url - parent_id = azurerm_user_assigned_identity.workload.id subject = "system:serviceaccount:default:magic8ball-sa" } @@ -99,11 +98,6 @@ resource "azurerm_cognitive_account" "openai" { kind = "OpenAI" custom_subdomain_name = "magic8ball-${local.random_id}" sku_name = "S0" - public_network_access_enabled = true - - identity { - type = "SystemAssigned" - } } resource "azurerm_cognitive_deployment" "deployment" { @@ -121,6 +115,15 @@ resource "azurerm_cognitive_deployment" "deployment" { } } +resource "azurerm_role_assignment" "cognitive_services_user" { + scope = azurerm_cognitive_account.openai.id + role_definition_name = "Cognitive Services OpenAI Contributor" + principal_id = azurerm_user_assigned_identity.workload.principal_id + principal_type = "ServicePrincipal" + + skip_service_principal_aad_check = true +} + ############################################################################### # Networking ############################################################################### diff --git a/scenarios/AksOpenAiTerraform/infra/outputs.tf b/scenarios/AksOpenAiTerraform/infra/outputs.tf index 29fc697ff..9bc08a64b 100644 --- a/scenarios/AksOpenAiTerraform/infra/outputs.tf +++ b/scenarios/AksOpenAiTerraform/infra/outputs.tf @@ -12,4 +12,8 @@ output "acr_login_url" { output "openai_endpoint" { value = azurerm_cognitive_account.openai.endpoint +} + +output "openai_deployment" { + value = azurerm_cognitive_deployment.deployment.name } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index 712f1f26d..b1a899b75 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -6,11 +6,10 @@ from azure.identity import DefaultAzureCredential, get_bearer_token_provider deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") -api_version = os.environ.get("AZURE_OPENAI_VERSION") azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") client = AzureOpenAI( - api_version=api_version, + api_version="2024-10-21", azure_endpoint=azure_endpoint, azure_ad_token_provider=get_bearer_token_provider( DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default" diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index 668267a70..6e6cc4fe0 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -4,9 +4,7 @@ metadata: name: magic8ball-configmap data: AZURE_OPENAI_ENDPOINT: $AZURE_OPENAI_ENDPOINT - AZURE_OPENAI_MODEL: $AZURE_OPENAI_MODEL - AZURE_OPENAI_DEPLOYMENT: $AZURE_OPENAI_MODEL - AZURE_OPENAI_VERSION: $AZURE_OPENAI_VERSION + AZURE_OPENAI_DEPLOYMENT: $AZURE_OPENAI_DEPLOYMENT --- apiVersion: apps/v1 kind: Deployment From 442fdc8de68081e5006ea982e7a9977f04fc4a16 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 27 Feb 2025 15:06:37 -0500 Subject: [PATCH 099/119] Minor tweak --- scenarios/AksOpenAiTerraform/magic8ball/app.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index b1a899b75..e6181176d 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -77,10 +77,12 @@ def disable_chat(): with st.chat_message("user"): st.write(prompt) - # Print Response + # Loading indicator response = None - with st.spinner("Loading response..."): # Loading indicator + with st.spinner("Loading response..."): response = call_api(st.session_state.messages) + + # Print Response st.session_state.messages.append({"role": "assistant", "content": response}) with st.chat_message("assistant"): st.write(response) From b8099ad0207cd501aca2f835aada22241b5e414a Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 27 Feb 2025 18:14:49 -0500 Subject: [PATCH 100/119] Fixes --- scenarios/AksOpenAiTerraform/README.md | 2 +- scenarios/AksOpenAiTerraform/infra/main.tf | 11 ++++++----- scenarios/AksOpenAiTerraform/magic8ball/app.py | 6 +++--- scenarios/AksOpenAiTerraform/quickstart-app.yml | 9 +++++---- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index 33bfc52d7..c883dddfc 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -8,7 +8,7 @@ ms.author: ariaamini ms.custom: innovation-engine, linux-related-content --- -## Provision Resources +## Provision Resources (~10 minutes) Run terraform to provision all the required Azure resources ```bash # Terraform parses TF_VAR_* (Ex: TF_VAR_xname -> xname) diff --git a/scenarios/AksOpenAiTerraform/infra/main.tf b/scenarios/AksOpenAiTerraform/infra/main.tf index 1eb39783a..6c422bd94 100644 --- a/scenarios/AksOpenAiTerraform/infra/main.tf +++ b/scenarios/AksOpenAiTerraform/infra/main.tf @@ -31,11 +31,12 @@ resource "azurerm_kubernetes_cluster" "main" { name = "AksCluster" location = var.location resource_group_name = azurerm_resource_group.main.name - + sku_tier = "Standard" - kubernetes_version = var.kubernetes_version dns_prefix = "AksCluster${local.random_id}" + kubernetes_version = var.kubernetes_version automatic_upgrade_channel = "stable" + workload_identity_enabled = true oidc_issuer_enabled = true @@ -45,7 +46,7 @@ resource "azurerm_kubernetes_cluster" "main" { default_node_pool { name = "agentpool" vm_size = "Standard_DS2_v2" - node_count = 2 + node_count = 1 upgrade_settings { max_surge = "10%" @@ -63,7 +64,7 @@ resource "azurerm_kubernetes_cluster" "main" { resource "azurerm_kubernetes_cluster_node_pool" "this" { name = "userpool" mode = "User" - node_count = 2 + node_count = 1 kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id orchestrator_version = var.kubernetes_version @@ -117,7 +118,7 @@ resource "azurerm_cognitive_deployment" "deployment" { resource "azurerm_role_assignment" "cognitive_services_user" { scope = azurerm_cognitive_account.openai.id - role_definition_name = "Cognitive Services OpenAI Contributor" + role_definition_name = "Cognitive Services User" principal_id = azurerm_user_assigned_identity.workload.principal_id principal_type = "ServicePrincipal" diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index e6181176d..937474fc0 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -5,7 +5,7 @@ import streamlit as st from azure.identity import DefaultAzureCredential, get_bearer_token_provider -deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") +azure_deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") client = AzureOpenAI( @@ -19,8 +19,8 @@ def call_api(messages): completion = client.chat.completions.create( - model=deployment, messages=messages, + model=azure_deployment ) return completion.choices[0].message.content @@ -79,7 +79,7 @@ def disable_chat(): # Loading indicator response = None - with st.spinner("Loading response..."): + with st.spinner("Loading response..."): response = call_api(st.session_state.messages) # Print Response diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index 6e6cc4fe0..bfac02181 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -12,7 +12,6 @@ metadata: name: magic8ball labels: app.kubernetes.io/name: magic8ball - azure.workload.identity/use: "true" spec: replicas: 3 selector: @@ -22,6 +21,7 @@ spec: metadata: labels: app.kubernetes.io/name: magic8ball + azure.workload.identity/use: "true" spec: serviceAccountName: magic8ball-sa containers: @@ -37,11 +37,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: magic8ball-service + name: magic8ball spec: + type: LoadBalancer selector: app.kubernetes.io/name: magic8ball - type: LoadBalancer ports: - protocol: TCP port: 80 @@ -52,4 +52,5 @@ kind: ServiceAccount metadata: name: magic8ball-sa annotations: - azure.workload.identity/client-id: $WORKLOAD_IDENTITY_CLIENT_ID \ No newline at end of file + azure.workload.identity/client-id: $WORKLOAD_IDENTITY_CLIENT_ID + azure.workload.identity/tenant-id: $TENANT_ID \ No newline at end of file From 54e909faebfd909ee6c3a1d7c8cc7f84900d24d9 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Thu, 27 Feb 2025 18:14:56 -0500 Subject: [PATCH 101/119] Remove unused resources --- scenarios/AksOpenAiTerraform/infra/main.tf | 39 ---------------------- 1 file changed, 39 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/infra/main.tf b/scenarios/AksOpenAiTerraform/infra/main.tf index 6c422bd94..aca86d3db 100644 --- a/scenarios/AksOpenAiTerraform/infra/main.tf +++ b/scenarios/AksOpenAiTerraform/infra/main.tf @@ -163,29 +163,6 @@ resource "azurerm_bastion_host" "this" { } } -############################################################################### -# Key Vault -############################################################################### -resource "azurerm_key_vault" "this" { - name = "KeyVault${local.random_id}" - location = var.location - resource_group_name = azurerm_resource_group.main.name - tenant_id = local.tenant_id - - sku_name = "standard" - enabled_for_deployment = true - enabled_for_disk_encryption = true - enabled_for_template_deployment = true - enable_rbac_authorization = true - purge_protection_enabled = false - soft_delete_retention_days = 30 - - network_acls { - bypass = "AzureServices" - default_action = "Allow" - } -} - ############################################################################### # Container Registry ############################################################################### @@ -195,20 +172,4 @@ resource "azurerm_container_registry" "this" { location = var.location sku = "Premium" anonymous_pull_enabled = true -} - -############################################################################### -# Storage Account -############################################################################### -resource "azurerm_storage_account" "storage_account" { - name = "boot${local.random_id}" - location = var.location - resource_group_name = azurerm_resource_group.main.name - - account_kind = "StorageV2" - account_tier = "Standard" - account_replication_type = "LRS" - is_hns_enabled = false - - allow_nested_items_to_be_public = false } \ No newline at end of file From 69124bdbb05d6328a7bd667ebb67e102b99e3ca1 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Fri, 28 Feb 2025 15:08:20 -0500 Subject: [PATCH 102/119] Rename --- .../AksOpenAiTerraform/{infra => terraform}/.terraform.lock.hcl | 0 scenarios/AksOpenAiTerraform/{infra => terraform}/main.tf | 0 scenarios/AksOpenAiTerraform/{infra => terraform}/outputs.tf | 0 scenarios/AksOpenAiTerraform/{infra => terraform}/provider.tf | 0 scenarios/AksOpenAiTerraform/{infra => terraform}/variables.tf | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename scenarios/AksOpenAiTerraform/{infra => terraform}/.terraform.lock.hcl (100%) rename scenarios/AksOpenAiTerraform/{infra => terraform}/main.tf (100%) rename scenarios/AksOpenAiTerraform/{infra => terraform}/outputs.tf (100%) rename scenarios/AksOpenAiTerraform/{infra => terraform}/provider.tf (100%) rename scenarios/AksOpenAiTerraform/{infra => terraform}/variables.tf (100%) diff --git a/scenarios/AksOpenAiTerraform/infra/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl similarity index 100% rename from scenarios/AksOpenAiTerraform/infra/.terraform.lock.hcl rename to scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl diff --git a/scenarios/AksOpenAiTerraform/infra/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf similarity index 100% rename from scenarios/AksOpenAiTerraform/infra/main.tf rename to scenarios/AksOpenAiTerraform/terraform/main.tf diff --git a/scenarios/AksOpenAiTerraform/infra/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf similarity index 100% rename from scenarios/AksOpenAiTerraform/infra/outputs.tf rename to scenarios/AksOpenAiTerraform/terraform/outputs.tf diff --git a/scenarios/AksOpenAiTerraform/infra/provider.tf b/scenarios/AksOpenAiTerraform/terraform/provider.tf similarity index 100% rename from scenarios/AksOpenAiTerraform/infra/provider.tf rename to scenarios/AksOpenAiTerraform/terraform/provider.tf diff --git a/scenarios/AksOpenAiTerraform/infra/variables.tf b/scenarios/AksOpenAiTerraform/terraform/variables.tf similarity index 100% rename from scenarios/AksOpenAiTerraform/infra/variables.tf rename to scenarios/AksOpenAiTerraform/terraform/variables.tf From f5ea6c2ee3f40e56a0f2db219533e849b81b5832 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Fri, 28 Feb 2025 16:10:31 -0500 Subject: [PATCH 103/119] Fixes --- scenarios/AksOpenAiTerraform/README.md | 39 +++++++++---------- .../AksOpenAiTerraform/magic8ball/app.py | 9 +++-- .../AksOpenAiTerraform/quickstart-app.yml | 1 + .../terraform/.terraform.lock.hcl | 28 ++++++------- .../AksOpenAiTerraform/terraform/main.tf | 28 +++++++------ .../AksOpenAiTerraform/terraform/provider.tf | 12 ------ 6 files changed, 55 insertions(+), 62 deletions(-) delete mode 100644 scenarios/AksOpenAiTerraform/terraform/provider.tf diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index c883dddfc..fa4a2ec74 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -8,44 +8,41 @@ ms.author: ariaamini ms.custom: innovation-engine, linux-related-content --- -## Provision Resources (~10 minutes) -Run terraform to provision all the required Azure resources +## Provision Resources with Terraform (~8 minutes) +Run terraform to provision all the Azure resources required to setup your new OpenAI website. ```bash -# Terraform parses TF_VAR_* (Ex: TF_VAR_xname -> xname) +# Terraform parses TF_VAR_* as vars (Ex: TF_VAR_name -> name) export TF_VAR_location="westus3" export TF_VAR_kubernetes_version="1.30.7" export TF_VAR_model_name="gpt-4o-mini" export TF_VAR_model_version="2024-07-18" - -terraform -chdir=infra init -terraform -chdir=infra apply -auto-approve +# Terraform consumes sub id as $ARM_SUBSCRIPTION_ID +export ARM_SUBSCRIPTION_ID=$SUBSCRIPTION_ID +# Run Terraform +terraform -chdir=terraform init +terraform -chdir=terraform apply -auto-approve ``` ## Login to Cluster +In order to use the kubectl to run commands on the newly created cluster, you must first login. ```bash -RESOURCE_GROUP=$(terraform -chdir=infra output -raw resource_group_name) +RESOURCE_GROUP=$(terraform -chdir=terraform output -raw resource_group_name) az aks get-credentials --admin --name AksCluster --resource-group $RESOURCE_GROUP --subscription $SUBSCRIPTION_ID ``` ## Deploy +Apply/Deploy Manifest File ```bash -## Build Dockerfile -ACR_LOGIN_URL=$(terraform -chdir=infra output -raw acr_login_url) -IMAGE="$ACR_LOGIN_URL/magic8ball:v1" -az acr login --name $ACR_LOGIN_URL -docker build -t $IMAGE ./magic8ball --push - -# Apply Manifest File -export IMAGE -export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform -chdir=infra output -raw workload_identity_client_id) -export AZURE_OPENAI_DEPLOYMENT=$(terraform -chdir=infra output -raw openai_deployment) -export AZURE_OPENAI_ENDPOINT=$(terraform -chdir=infra output -raw openai_endpoint) -envsubst < quickstart-app.yml | kubectl apply -f -``` +export IMAGE="aamini8/magic8ball:v1" +export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform -chdir=terraform output -raw workload_identity_client_id) +export AZURE_OPENAI_DEPLOYMENT=$(terraform -chdir=terraform output -raw openai_deployment) +export AZURE_OPENAI_ENDPOINT=$(terraform -chdir=terraform output -raw openai_endpoint) +envsubst < quickstart-app.yml | kubectl apply -f - ``` ## Wait for public IP ```bash -kubectl wait --for=jsonpath="{.status.loadBalancer.ingress[0].ip}" service/magic8ball-service -PUBLIC_IP=$(kubectl get service/magic8ball-service -o=jsonpath="{.status.loadBalancer.ingress[0].ip}") +kubectl wait --for=jsonpath="{.status.loadBalancer.ingress[0].ip}" service/magic8ball +PUBLIC_IP=$(kubectl get service/magic8ball -o=jsonpath="{.status.loadBalancer.ingress[0].ip}") echo "Connect to app: $PUBLIC_IP" ``` \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index 937474fc0..cb3efe578 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -3,24 +3,25 @@ import os from openai import AzureOpenAI import streamlit as st -from azure.identity import DefaultAzureCredential, get_bearer_token_provider +from azure.identity import WorkloadIdentityCredential, get_bearer_token_provider azure_deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") +workload_identity_client_id = os.getenv("WORKLOAD_IDENTITY_CLIENT_ID") client = AzureOpenAI( api_version="2024-10-21", azure_endpoint=azure_endpoint, azure_ad_token_provider=get_bearer_token_provider( - DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default" + WorkloadIdentityCredential(client_id=workload_identity_client_id), + "https://cognitiveservices.azure.com/.default", ), ) def call_api(messages): completion = client.chat.completions.create( - messages=messages, - model=azure_deployment + messages=messages, model=azure_deployment ) return completion.choices[0].message.content diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index bfac02181..50a2abc28 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -5,6 +5,7 @@ metadata: data: AZURE_OPENAI_ENDPOINT: $AZURE_OPENAI_ENDPOINT AZURE_OPENAI_DEPLOYMENT: $AZURE_OPENAI_DEPLOYMENT + WORKLOAD_IDENTITY_CLIENT_ID: $WORKLOAD_IDENTITY_CLIENT_ID --- apiVersion: apps/v1 kind: Deployment diff --git a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl index 6222f4e7e..3ea2ce44c 100644 --- a/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl +++ b/scenarios/AksOpenAiTerraform/terraform/.terraform.lock.hcl @@ -2,22 +2,22 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/azurerm" { - version = "4.16.0" - constraints = "~> 4.16.0" + version = "4.20.0" + constraints = "~> 4.20.0" hashes = [ - "h1:7e25Wr4cpUvlAcwL+9ZOeeA1xha84LqTZNviDaVQFlo=", - "zh:2035e461a94bd4180557a06f8e56f228a8a035608d0dac4d08e5870cf9265276", - "zh:3f15778a22ef1b9d0fa28670e5ea6ef1094b0be2533f43f350a2ef15d471b353", - "zh:4f1a4d03b008dd958bcd6bf82cf088fbaa9c121be2fd35e10e6b06c6e8f6aaa1", - "zh:5859f31c342364e849b4f8c437a46f33e927fa820244d0732b8d2ec74a95712d", - "zh:693d0f15512ca8c6b5e999b3a7551503feb06b408b3836bc6a6403e518b9ddab", - "zh:7f4912bec5b04f5156935292377c12484c13582151eb3c2555df409a7e5fb6e0", - "zh:bb9a509497f3a131c52fac32348919bf1b9e06c69a65f24607b03f7b56fb47b6", - "zh:c1b0c64e49ac591fd038ad71e71403ff71c07476e27e8da718c29f0028ea6d0d", - "zh:dd4ca432ee14eb0bb0cdc0bb463c8675b8ef02497be870a20d8dfee3e7fe52b3", - "zh:df58bb7fea984d2b11709567842ca4d55b3f24e187aa6be99e3677f55cbbe7da", + "h1:O7hZA85M9/G5LZt+m0bppCinoyp8C346JpI+QnMjYVo=", + "zh:0d29f06abed90da7b943690244420fe1de3e28d4c6de0db441f1af2aa91ea6b8", + "zh:2345e07e91dfec9af3df25fd5119d3a09f91e37ca10af30a344f7b3c297e9ad8", + "zh:42d77650df0238333bcce5da91b4f3d62e54b1ed456f58a9c913270d80a70262", + "zh:43ce137f2644769ceada99a2c815c9c30807e42f61f2f6ce60869411217375f9", + "zh:5e4d8f6a5212f6b7ba29846a2ff328214c7f983ce772196f8e6721edcefd4c59", + "zh:69613d671884fc568a075359e2920d7c19e6d588717b4532b90fb4a4ca8aabd0", + "zh:827ca4fcc25958c731677cb1d87cb09764e3a24ae4117fd9776429341fcdeabe", + "zh:8fad25f949dff7c6f40ea22b13a8b4de6ea0de3c5a975c4a3281529e4797e897", + "zh:b3d175e2725fe38f2a71d5fb346a9d4ff70d449a9d229c95c24f88e764dd2d47", + "zh:c53f3fef67aa64664c85bb8603b0a9730a267a76d7d84ceae16416de7ccb2437", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f7fb37704da50c096f9c7c25e8a95fe73ce1d3c5aab0d616d506f07bc5cfcdd8", + "zh:f7d9ff06344547232e6c84bc3f6bf9c29cf978ba7cd585c10f4c3361a4b81f22", ] } diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index aca86d3db..9318defec 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -1,3 +1,20 @@ +############################################################################### +# Plugin setup +############################################################################### +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.20.0" + } + } +} + +provider "azurerm" { + features {} +} +############################################################################### + data "azurerm_client_config" "current" { } @@ -161,15 +178,4 @@ resource "azurerm_bastion_host" "this" { subnet_id = azurerm_subnet.this.id public_ip_address_id = azurerm_public_ip.this.id } -} - -############################################################################### -# Container Registry -############################################################################### -resource "azurerm_container_registry" "this" { - name = "acr${local.random_id}" - resource_group_name = azurerm_resource_group.main.name - location = var.location - sku = "Premium" - anonymous_pull_enabled = true } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/provider.tf b/scenarios/AksOpenAiTerraform/terraform/provider.tf deleted file mode 100644 index 5d9512e59..000000000 --- a/scenarios/AksOpenAiTerraform/terraform/provider.tf +++ /dev/null @@ -1,12 +0,0 @@ -terraform { - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "~> 4.16.0" - } - } -} - -provider "azurerm" { - features {} -} \ No newline at end of file From 74780d57c96f704c0b2132f1faabb40b8c999f67 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Sat, 1 Mar 2025 03:06:28 -0500 Subject: [PATCH 104/119] Fixes --- scenarios/AksOpenAiTerraform/README.md | 24 +++++++++- .../AksOpenAiTerraform/quickstart-app.yml | 46 +++++++++++++++++-- .../AksOpenAiTerraform/terraform/main.tf | 31 +------------ .../AksOpenAiTerraform/terraform/outputs.tf | 16 +++++-- 4 files changed, 79 insertions(+), 38 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index fa4a2ec74..ee691f11a 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -13,7 +13,7 @@ Run terraform to provision all the Azure resources required to setup your new Op ```bash # Terraform parses TF_VAR_* as vars (Ex: TF_VAR_name -> name) export TF_VAR_location="westus3" -export TF_VAR_kubernetes_version="1.30.7" +export TF_VAR_kubernetes_version="1.30.9" export TF_VAR_model_name="gpt-4o-mini" export TF_VAR_model_version="2024-07-18" # Terraform consumes sub id as $ARM_SUBSCRIPTION_ID @@ -30,6 +30,27 @@ RESOURCE_GROUP=$(terraform -chdir=terraform output -raw resource_group_name) az aks get-credentials --admin --name AksCluster --resource-group $RESOURCE_GROUP --subscription $SUBSCRIPTION_ID ``` +# Install Helm Charts +Install Prometheus, nginx-ingress, and cert-manager +```bash +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo add jetstack https://charts.jetstack.io +helm repo update + +export HOSTNAME=$(terraform -chdir=terraform output -raw hostname) +export STATIC_IP=$(terraform -chdir=terraform output -raw static_ip) +helm install ingress-nginx ingress-nginx/ingress-nginx \ + --set controller.replicaCount=2 \ + --set controller.nodeSelector."kubernetes\.io/os"=linux \ + --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-dns-label-name"=$HOSTNAME \ + --set controller.service.loadBalancerIP=$STATIC_IP \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz +helm install cert-manager jetstack/cert-manager \ + --set crds.enabled=true \ + --set nodeSelector."kubernetes\.io/os"=linux +``` + ## Deploy Apply/Deploy Manifest File ```bash @@ -37,6 +58,7 @@ export IMAGE="aamini8/magic8ball:v1" export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform -chdir=terraform output -raw workload_identity_client_id) export AZURE_OPENAI_DEPLOYMENT=$(terraform -chdir=terraform output -raw openai_deployment) export AZURE_OPENAI_ENDPOINT=$(terraform -chdir=terraform output -raw openai_endpoint) +export DNS_LABEL=$(terraform -chdir=terraform output -raw dns_label) envsubst < quickstart-app.yml | kubectl apply -f - ``` diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index 50a2abc28..7b41925a8 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -40,13 +40,12 @@ kind: Service metadata: name: magic8ball spec: - type: LoadBalancer selector: app.kubernetes.io/name: magic8ball ports: - - protocol: TCP - port: 80 + - port: 80 targetPort: 8501 + protocol: TCP --- apiVersion: v1 kind: ServiceAccount @@ -54,4 +53,43 @@ metadata: name: magic8ball-sa annotations: azure.workload.identity/client-id: $WORKLOAD_IDENTITY_CLIENT_ID - azure.workload.identity/tenant-id: $TENANT_ID \ No newline at end of file + azure.workload.identity/tenant-id: $TENANT_ID +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: magic8ball + annotations: + cert-manager.io/issuer: letsencrypt-dev +spec: + ingressClassName: nginx + tls: + - hosts: + - $HOSTNAME + secretName: tls-secret + rules: + - host: $HOSTNAME + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: magic8ball + port: + number: 80 +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: letsencrypt-dev +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: $EMAIL + privateKeySecretRef: + name: tls-secret + solvers: + - http01: + ingress: + ingressClassName: nginx \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 9318defec..651b3f6aa 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -145,37 +145,10 @@ resource "azurerm_role_assignment" "cognitive_services_user" { ############################################################################### # Networking ############################################################################### -resource "azurerm_virtual_network" "this" { - name = "Vnet" - location = var.location - resource_group_name = azurerm_resource_group.main.name - - address_space = ["10.0.0.0/8"] -} - -resource "azurerm_subnet" "this" { - name = "AzureBastionSubnet" - resource_group_name = azurerm_resource_group.main.name - - virtual_network_name = azurerm_virtual_network.this.name - address_prefixes = ["10.243.2.0/24"] -} - resource "azurerm_public_ip" "this" { name = "PublicIp" + domain_name_label = "magic8ball-${local.random_id}" location = var.location - resource_group_name = azurerm_resource_group.main.name + resource_group_name = azurerm_kubernetes_cluster.main.node_resource_group allocation_method = "Static" -} - -resource "azurerm_bastion_host" "this" { - name = "BastionHost" - location = var.location - resource_group_name = azurerm_resource_group.main.name - - ip_configuration { - name = "configuration" - subnet_id = azurerm_subnet.this.id - public_ip_address_id = azurerm_public_ip.this.id - } } \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/terraform/outputs.tf b/scenarios/AksOpenAiTerraform/terraform/outputs.tf index 9bc08a64b..2411dcba1 100644 --- a/scenarios/AksOpenAiTerraform/terraform/outputs.tf +++ b/scenarios/AksOpenAiTerraform/terraform/outputs.tf @@ -6,14 +6,22 @@ output "workload_identity_client_id" { value = azurerm_user_assigned_identity.workload.client_id } -output "acr_login_url" { - value = azurerm_container_registry.this.login_server -} - output "openai_endpoint" { value = azurerm_cognitive_account.openai.endpoint } output "openai_deployment" { value = azurerm_cognitive_deployment.deployment.name +} + +output "hostname" { + value = azurerm_public_ip.this.fqdn +} + +output "static_ip" { + value = azurerm_public_ip.this.ip_address +} + +output "dns_label" { + value = azurerm_public_ip.this.domain_name_label } \ No newline at end of file From 24339d8eec24deb50c01f317477f614c3911fd5c Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Sat, 1 Mar 2025 03:18:18 -0500 Subject: [PATCH 105/119] fix --- scenarios/AksOpenAiTerraform/terraform/main.tf | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index 651b3f6aa..f0207b10d 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -1,5 +1,5 @@ ############################################################################### -# Plugin setup +# azurerm plugin setup ############################################################################### terraform { required_providers { @@ -13,8 +13,10 @@ terraform { provider "azurerm" { features {} } -############################################################################### +############################################################################### +# Resource Group +############################################################################### data "azurerm_client_config" "current" { } @@ -78,18 +80,6 @@ resource "azurerm_kubernetes_cluster" "main" { } } -resource "azurerm_kubernetes_cluster_node_pool" "this" { - name = "userpool" - mode = "User" - node_count = 1 - - kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id - orchestrator_version = var.kubernetes_version - vm_size = "Standard_DS2_v2" - os_type = "Linux" - priority = "Regular" -} - resource "azurerm_user_assigned_identity" "workload" { name = "WorkloadManagedIdentity" resource_group_name = azurerm_resource_group.main.name From a87162bea5e4edf0eb57c58d3e06f25bcca010c2 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Sat, 1 Mar 2025 03:44:59 -0500 Subject: [PATCH 106/119] Fix --- scenarios/AksOpenAiTerraform/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index ee691f11a..ab1ba719b 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -31,22 +31,22 @@ az aks get-credentials --admin --name AksCluster --resource-group $RESOURCE_GROU ``` # Install Helm Charts -Install Prometheus, nginx-ingress, and cert-manager +Install nginx and cert-manager ```bash helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx helm repo add jetstack https://charts.jetstack.io helm repo update -export HOSTNAME=$(terraform -chdir=terraform output -raw hostname) -export STATIC_IP=$(terraform -chdir=terraform output -raw static_ip) -helm install ingress-nginx ingress-nginx/ingress-nginx \ +STATIC_IP=$(terraform -chdir=terraform output -raw static_ip) +DNS_LABEL=$(terraform -chdir=terraform output -raw dns_label) +helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx \ --set controller.replicaCount=2 \ --set controller.nodeSelector."kubernetes\.io/os"=linux \ --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \ - --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-dns-label-name"=$HOSTNAME \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-dns-label-name"=$DNS_LABEL \ --set controller.service.loadBalancerIP=$STATIC_IP \ --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz -helm install cert-manager jetstack/cert-manager \ +helm upgrade --install cert-manager jetstack/cert-manager \ --set crds.enabled=true \ --set nodeSelector."kubernetes\.io/os"=linux ``` @@ -55,16 +55,16 @@ helm install cert-manager jetstack/cert-manager \ Apply/Deploy Manifest File ```bash export IMAGE="aamini8/magic8ball:v1" +export HOSTNAME=$(terraform -chdir=terraform output -raw hostname) export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform -chdir=terraform output -raw workload_identity_client_id) export AZURE_OPENAI_DEPLOYMENT=$(terraform -chdir=terraform output -raw openai_deployment) export AZURE_OPENAI_ENDPOINT=$(terraform -chdir=terraform output -raw openai_endpoint) -export DNS_LABEL=$(terraform -chdir=terraform output -raw dns_label) envsubst < quickstart-app.yml | kubectl apply -f - ``` ## Wait for public IP ```bash kubectl wait --for=jsonpath="{.status.loadBalancer.ingress[0].ip}" service/magic8ball -PUBLIC_IP=$(kubectl get service/magic8ball -o=jsonpath="{.status.loadBalancer.ingress[0].ip}") +PUBLIC_IP=$(kubectl get service ingress-nginx-controller -o=jsonpath="{.status.loadBalancer.ingress[0].ip}") echo "Connect to app: $PUBLIC_IP" ``` \ No newline at end of file From a3450bb73013d70af1e2833bd70e2cb1c2c15143 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Sat, 1 Mar 2025 03:48:51 -0500 Subject: [PATCH 107/119] Fix --- scenarios/AksOpenAiTerraform/quickstart-app.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index 7b41925a8..698547a94 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -68,8 +68,7 @@ spec: - $HOSTNAME secretName: tls-secret rules: - - host: $HOSTNAME - http: + - http: paths: - path: / pathType: Prefix From 55ae013a14f055d60725ae50397812261c572982 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Sat, 1 Mar 2025 03:54:20 -0500 Subject: [PATCH 108/119] Fix --- scenarios/AksOpenAiTerraform/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index ab1ba719b..a0babe55b 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -64,7 +64,7 @@ envsubst < quickstart-app.yml | kubectl apply -f - ## Wait for public IP ```bash -kubectl wait --for=jsonpath="{.status.loadBalancer.ingress[0].ip}" service/magic8ball -PUBLIC_IP=$(kubectl get service ingress-nginx-controller -o=jsonpath="{.status.loadBalancer.ingress[0].ip}") -echo "Connect to app: $PUBLIC_IP" +kubectl wait --for=jsonpath="{.status.loadBalancer.ingress[0].ip}" service/ingress-nginx-controller +PUBLIC_IP=$(kubectl get service/ingress-nginx-controller -o=jsonpath="{.status.loadBalancer.ingress[0].ip}") +echo "Visit: https://$HOSTNAME (IP Address: $PUBLIC_IP)" ``` \ No newline at end of file From b65a2b6cf926cf9990970a4464a55da5d3906558 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 3 Mar 2025 15:57:04 -0500 Subject: [PATCH 109/119] Fix --- scenarios/AksOpenAiTerraform/README.md | 2 +- scenarios/AksOpenAiTerraform/quickstart-app.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index a0babe55b..aea9b9da0 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -66,5 +66,5 @@ envsubst < quickstart-app.yml | kubectl apply -f - ```bash kubectl wait --for=jsonpath="{.status.loadBalancer.ingress[0].ip}" service/ingress-nginx-controller PUBLIC_IP=$(kubectl get service/ingress-nginx-controller -o=jsonpath="{.status.loadBalancer.ingress[0].ip}") -echo "Visit: https://$HOSTNAME (IP Address: $PUBLIC_IP)" +echo "Visit: https://$HOSTNAME" ``` \ No newline at end of file diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index 698547a94..7b41925a8 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -68,7 +68,8 @@ spec: - $HOSTNAME secretName: tls-secret rules: - - http: + - host: $HOSTNAME + http: paths: - path: / pathType: Prefix From 4a96ec711b783ca0074c16ef3e075d98b7db2fa6 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 3 Mar 2025 17:44:35 -0500 Subject: [PATCH 110/119] Reduce replica count --- scenarios/AksOpenAiTerraform/quickstart-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index 7b41925a8..cf465e374 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -14,7 +14,7 @@ metadata: labels: app.kubernetes.io/name: magic8ball spec: - replicas: 3 + replicas: 1 selector: matchLabels: app.kubernetes.io/name: magic8ball From 6ad16bb7ad5a5cfa50dc3bcfdef2912c50c8a3de Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 3 Mar 2025 17:44:50 -0500 Subject: [PATCH 111/119] Update prompt --- scenarios/AksOpenAiTerraform/magic8ball/app.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index cb3efe578..d719a93c3 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -27,7 +27,7 @@ def call_api(messages): assistant_prompt = """ -You are the infamous Magic 8 Ball. You need to randomly reply to any question with one of the following answers: +You are the famous Magic 8 Ball. You need to randomly reply to any question with one of the following answers: - It is certain. - It is decidedly so. @@ -39,7 +39,6 @@ def call_api(messages): - Outlook good. - Yes. - Signs point to yes. -- Reply hazy, try again. - Ask again later. - Better not tell you now. - Cannot predict now. @@ -50,8 +49,7 @@ def call_api(messages): - Outlook not so good. - Very doubtful. -Add a short comment in a pirate style at the end! Follow your heart and be creative! -For mor information, see https://en.wikipedia.org/wiki/Magic_8_Ball +If the question the user provides is unclear, remind them: "Ask the magic8ball any question and I will predict your future!" """ # Init state From 3f2ad290160ed50aefc12c8ed7a4783e46674e95 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 3 Mar 2025 17:44:58 -0500 Subject: [PATCH 112/119] Update version --- scenarios/AksOpenAiTerraform/magic8ball/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/magic8ball/Dockerfile b/scenarios/AksOpenAiTerraform/magic8ball/Dockerfile index 68dcce690..fe9aa8ca5 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/Dockerfile +++ b/scenarios/AksOpenAiTerraform/magic8ball/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.13-slim WORKDIR /app ENV PYTHONDONTWRITEBYTECODE=1 From f32e4fca3fad19e6d41b905d8b115c7dcf12ffda Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 3 Mar 2025 17:54:48 -0500 Subject: [PATCH 113/119] Update eta --- scenarios/AksOpenAiTerraform/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index aea9b9da0..5a9efe988 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -8,7 +8,7 @@ ms.author: ariaamini ms.custom: innovation-engine, linux-related-content --- -## Provision Resources with Terraform (~8 minutes) +## Provision Resources with Terraform (~5 minutes) Run terraform to provision all the Azure resources required to setup your new OpenAI website. ```bash # Terraform parses TF_VAR_* as vars (Ex: TF_VAR_name -> name) @@ -31,7 +31,7 @@ az aks get-credentials --admin --name AksCluster --resource-group $RESOURCE_GROU ``` # Install Helm Charts -Install nginx and cert-manager +Install nginx and cert-manager through Helm ```bash helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx helm repo add jetstack https://charts.jetstack.io @@ -55,6 +55,8 @@ helm upgrade --install cert-manager jetstack/cert-manager \ Apply/Deploy Manifest File ```bash export IMAGE="aamini8/magic8ball:v1" +# Uncomment below to manually build docker image yourself instead of using pre-built image. +# docker build -t ./magic8ball --push export HOSTNAME=$(terraform -chdir=terraform output -raw hostname) export WORKLOAD_IDENTITY_CLIENT_ID=$(terraform -chdir=terraform output -raw workload_identity_client_id) export AZURE_OPENAI_DEPLOYMENT=$(terraform -chdir=terraform output -raw openai_deployment) From fdc542058f771bc5164515d897f506fa791a24f0 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Mon, 3 Mar 2025 18:11:09 -0500 Subject: [PATCH 114/119] Fix --- scenarios/AksOpenAiTerraform/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index 5a9efe988..fc2e41aef 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -64,9 +64,8 @@ export AZURE_OPENAI_ENDPOINT=$(terraform -chdir=terraform output -raw openai_end envsubst < quickstart-app.yml | kubectl apply -f - ``` -## Wait for public IP +## Wait for host to be ready ```bash -kubectl wait --for=jsonpath="{.status.loadBalancer.ingress[0].ip}" service/ingress-nginx-controller -PUBLIC_IP=$(kubectl get service/ingress-nginx-controller -o=jsonpath="{.status.loadBalancer.ingress[0].ip}") +kubectl wait --for=condition=Ready certificate/tls-secret echo "Visit: https://$HOSTNAME" ``` \ No newline at end of file From 4f75a2503ca13048e0eb5fcbb9d0e1c31e5be42c Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 4 Mar 2025 01:56:09 -0500 Subject: [PATCH 115/119] Optimize app --- .../AksOpenAiTerraform/magic8ball/app.py | 41 ++++--------------- .../magic8ball/requirements.txt | 6 +-- 2 files changed, 11 insertions(+), 36 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index d719a93c3..15f32721f 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -19,37 +19,16 @@ ) -def call_api(messages): +def ask_openai_api(messages: list[str]): completion = client.chat.completions.create( - messages=messages, model=azure_deployment + messages=messages, model=azure_deployment, stream=True, max_tokens=20 ) - return completion.choices[0].message.content + return completion assistant_prompt = """ -You are the famous Magic 8 Ball. You need to randomly reply to any question with one of the following answers: - -- It is certain. -- It is decidedly so. -- Without a doubt. -- Yes definitely. -- You may rely on it. -- As I see it, yes. -- Most likely. -- Outlook good. -- Yes. -- Signs point to yes. -- Ask again later. -- Better not tell you now. -- Cannot predict now. -- Concentrate and ask again. -- Don't count on it. -- My reply is no. -- My sources say no. -- Outlook not so good. -- Very doubtful. - -If the question the user provides is unclear, remind them: "Ask the magic8ball any question and I will predict your future!" +Answer as a magic 8 ball and make random predictions. +If the question is not clear, respond with "Ask the Magic 8 Ball a question about your future." """ # Init state @@ -76,15 +55,11 @@ def disable_chat(): with st.chat_message("user"): st.write(prompt) - # Loading indicator - response = None - with st.spinner("Loading response..."): - response = call_api(st.session_state.messages) - # Print Response - st.session_state.messages.append({"role": "assistant", "content": response}) with st.chat_message("assistant"): - st.write(response) + messages = st.session_state.messages + response = st.write_stream(ask_openai_api(messages)) + st.session_state.messages.append({"role": "assistant", "content": response}) # Re-enable textbox st.session_state.disabled = False diff --git a/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt b/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt index 4e604dc3c..b32480fe0 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt +++ b/scenarios/AksOpenAiTerraform/magic8ball/requirements.txt @@ -1,3 +1,3 @@ -streamlit==1.40.1 -azure-identity==1.20.0 -openai==1.64.0 \ No newline at end of file +streamlit~=1.40.1 +azure-identity~=1.20.0 +openai~=1.65.2 \ No newline at end of file From b8eb0285c6b3235c21e4addda42dde1f5583e1fc Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 4 Mar 2025 02:02:13 -0500 Subject: [PATCH 116/119] Add spinner --- scenarios/AksOpenAiTerraform/magic8ball/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index 15f32721f..ce6d3fa46 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -58,7 +58,8 @@ def disable_chat(): # Print Response with st.chat_message("assistant"): messages = st.session_state.messages - response = st.write_stream(ask_openai_api(messages)) + with st.spinner("Loading..."): + response = st.write_stream(ask_openai_api(messages)) st.session_state.messages.append({"role": "assistant", "content": response}) # Re-enable textbox From dc7b464d748558ff034d6bc1b036596c2214fe9c Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 4 Mar 2025 02:23:51 -0500 Subject: [PATCH 117/119] Emoji --- scenarios/AksOpenAiTerraform/magic8ball/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index ce6d3fa46..3f75e3f7f 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -37,7 +37,7 @@ def ask_openai_api(messages: list[str]): if "disabled" not in st.session_state: st.session_state.disabled = False -st.title("Magic 8 Ball") +st.title(":robot_face: Magic 8 Ball") for message in st.session_state.messages[1:]: # Print previous messages with st.chat_message(message["role"]): st.markdown(message["content"]) From 0f6ba257a925a5896dd7ee63752ea2d480267eb4 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Tue, 4 Mar 2025 02:24:33 -0500 Subject: [PATCH 118/119] Remove comment --- scenarios/AksOpenAiTerraform/magic8ball/app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/magic8ball/app.py b/scenarios/AksOpenAiTerraform/magic8ball/app.py index 3f75e3f7f..a3de44d3a 100644 --- a/scenarios/AksOpenAiTerraform/magic8ball/app.py +++ b/scenarios/AksOpenAiTerraform/magic8ball/app.py @@ -1,5 +1,3 @@ -# https://levelup.gitconnected.com/its-time-to-create-a-private-chatgpt-for-yourself-today-6503649e7bb6 - import os from openai import AzureOpenAI import streamlit as st From fb40f0e25d60afa67aeb178aca7ff86729f9fdf7 Mon Sep 17 00:00:00 2001 From: Aria Amini Date: Wed, 5 Mar 2025 16:40:50 -0500 Subject: [PATCH 119/119] Minor tweaks --- scenarios/AksOpenAiTerraform/README.md | 2 +- scenarios/AksOpenAiTerraform/quickstart-app.yml | 2 +- scenarios/AksOpenAiTerraform/terraform/main.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scenarios/AksOpenAiTerraform/README.md b/scenarios/AksOpenAiTerraform/README.md index fc2e41aef..d6a9fbfcc 100644 --- a/scenarios/AksOpenAiTerraform/README.md +++ b/scenarios/AksOpenAiTerraform/README.md @@ -54,7 +54,7 @@ helm upgrade --install cert-manager jetstack/cert-manager \ ## Deploy Apply/Deploy Manifest File ```bash -export IMAGE="aamini8/magic8ball:v1" +export IMAGE="aamini8/magic8ball:latest" # Uncomment below to manually build docker image yourself instead of using pre-built image. # docker build -t ./magic8ball --push export HOSTNAME=$(terraform -chdir=terraform output -raw hostname) diff --git a/scenarios/AksOpenAiTerraform/quickstart-app.yml b/scenarios/AksOpenAiTerraform/quickstart-app.yml index cf465e374..0f2bb4854 100644 --- a/scenarios/AksOpenAiTerraform/quickstart-app.yml +++ b/scenarios/AksOpenAiTerraform/quickstart-app.yml @@ -14,7 +14,7 @@ metadata: labels: app.kubernetes.io/name: magic8ball spec: - replicas: 1 + replicas: 2 selector: matchLabels: app.kubernetes.io/name: magic8ball diff --git a/scenarios/AksOpenAiTerraform/terraform/main.tf b/scenarios/AksOpenAiTerraform/terraform/main.tf index f0207b10d..cf95667e4 100644 --- a/scenarios/AksOpenAiTerraform/terraform/main.tf +++ b/scenarios/AksOpenAiTerraform/terraform/main.tf @@ -65,7 +65,7 @@ resource "azurerm_kubernetes_cluster" "main" { default_node_pool { name = "agentpool" vm_size = "Standard_DS2_v2" - node_count = 1 + node_count = 2 upgrade_settings { max_surge = "10%"