Update CI to label-driven testing

.github/workflows/rocm-ci-dispatch.yml

@@ -0,0 +1,135 @@
+name: PR Automatic CI
+
+on:
+  pull_request:
+    branches:
+      - 'dev'
+      - 'release_v1.**_rocm'
+      - 'release_v2.**_rocm'
+    types: [ labeled, unlabeled, synchronize, reopened ]
+
+permissions:
+  contents: read
+  actions: write
+
+jobs:
+  determine_level:
+    runs-on: ubuntu-latest
+    outputs:
+      test_level: ${{ steps.set_level.outputs.test_level }}
+      has_level: ${{ steps.set_level.outputs.has_level }}
+      removed_level: ${{ steps.set_level.outputs.removed_level }}
+      removed_was_highest: ${{ steps.set_level.outputs.removed_was_highest }}
+    steps:
+      - name: Determine CI level from labels
+        id: set_level
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const labels = (context.payload.pull_request.labels || [])
+              .map(label => label.name.toLowerCase());
+
+            // Determine if a CI level label was removed and what level it was   
+            const removedLabel = (context.payload.label?.name || '').toLowerCase();
+            let removedLevel = 0;
+            if (removedLabel === 'ci-level 3') removedLevel = 3;
+            else if (removedLabel === 'ci-level 2') removedLevel = 2;
+            else if (removedLabel === 'ci-level 1') removedLevel = 1;
+
+            // Determine the current highest CI level from remaining labels
+            let level = '';
+            if (labels.includes('ci-level 3')) level = '3';
+            else if (labels.includes('ci-level 2')) level = '2';
+            else if (labels.includes('ci-level 1')) level = '1';
+
+            // Check if the removed level was higher than the current level
+            const currentLevel = level ? Number(level) : 0;
+            const removedWasHighest = removedLevel > 0 && removedLevel > currentLevel;
+
+            // Set outputs for use in later jobs
+            core.setOutput('test_level', level);
+            core.setOutput('has_level', level ? 'true' : 'false');
+            core.setOutput('removed_level', String(removedLevel));
+            core.setOutput('removed_was_highest', removedWasHighest ? 'true' : 'false');
+
+  cancel_others:
+    if: ${{ needs.determine_level.outputs.has_level == 'true' }}
+    needs: determine_level
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel queued/in-progress runs
+        uses: actions/github-script@v7
+        env:
+          CURRENT_LEVEL: ${{ needs.determine_level.outputs.test_level }}
+        with:
+          script: |
+            // Cancel other runs for the same PR that are queued or
+            // in progress and have a lower CI level
+            const prNumber = context.payload.pull_request.number;
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const currentRunId = context.runId;
+            const currentLevel = Number(process.env.CURRENT_LEVEL || 0);
+            const action = context.payload.action;
+
+            // If a label was added, determine its level to compare against other runs
+            const addedLabel = (context.payload.label?.name || '').toLowerCase();
+            let addedLevel = 0;
+            if (addedLabel === 'ci-level 3') addedLevel = 3;
+            else if (addedLabel === 'ci-level 2') addedLevel = 2;
+            else if (addedLabel === 'ci-level 1') addedLevel = 1;
+
+            // Fetch all workflow runs for this workflow and PR
+            const runs = await github.rest.actions.listWorkflowRuns({
+              owner,
+              repo,
+              workflow_id: 'rocm-ci-dispatch.yml',
+              event: 'pull_request',
+              per_page: 100,
+            });
+
+            // Filter runs to find those that are for the same PR,
+            // are queued or in progress, and have a lower CI level
+            const toCancel = runs.data.workflow_runs.filter(run => {
+              const prMatch = (run.pull_requests || []).some(pr => pr.number === prNumber);
+              const active = run.status === 'queued' || run.status === 'in_progress';
+              if (action === 'synchronize') {
+                return prMatch && active && run.id !== currentRunId;
+              }
+              // For labeled/unlabeled actions, compare the CI levels
+              const title = run.display_title || '';
+              const match = title.match(/CI Level (\d+)/i);
+              const runLevel = match ? Number(match[1]) : 0;
+              // If a label was added, cancel runs with a lower level than the added label
+              if (action === 'labeled') {
+                const isLowerThanAdded = runLevel > 0 && runLevel < addedLevel;
+                return prMatch && active && isLowerThanAdded && run.id !== currentRunId;
+              }
+              // If a label was removed, cancel runs with a lower level than the current level
+              const isLower = runLevel > 0 && runLevel < currentLevel;
+              return prMatch && active && isLower && run.id !== currentRunId;
+            });
+
+            // Cancel the identified runs
+            for (const run of toCancel) {
+              core.info(`Canceling dispatch run ${run.id} for PR #${prNumber}`);
+              await github.rest.actions.cancelWorkflowRun({
+                owner,
+                repo,
+                run_id: run.id,
+              });
+            }
+
+  dispatch:
+    # Run this job if there is a valid level to test (has_level == true),
+    # and any of the following are true:
+    # - A level label was added 
+    # - A commit was pushed
+    # - The PR was reopened
+    # - The highest level label was removed
+    if: ${{ needs.determine_level.outputs.has_level == 'true' && ((github.event.action != 'unlabeled') || (github.event.action == 'unlabeled' && needs.determine_level.outputs.removed_was_highest == 'true')) }}
+    needs: [determine_level, cancel_others]
+    name: CI Level ${{ needs.determine_level.outputs.test_level }}
+    uses: ./.github/workflows/rocm-ci.yml
+    with:
+      test_level: ${{ needs.determine_level.outputs.test_level }}

.github/workflows/rocm-ci.yml

@@ -2,19 +2,30 @@
 #
 # See LICENSE for license information.
 
-name: TransformerEngine CI
+name: Test and Build Branch
 
 on:
   push:
     branches:
       - 'dev'
       - 'release_v1.*_rocm'
       - 'release_v2.*_rocm'
-  pull_request:
-    branches:
-      - 'dev'
-      - 'release_v1.**_rocm'
-      - 'release_v2.**_rocm'
+  workflow_call:
+    inputs:
+      test_level:
+        description: 'Test Level (1-3)'
+        required: false
+        default: '1'
+        type: string
+      docker_image_override:
+        description: 'Manual Docker Image (Leave empty to use config file value)'
+        required: false
+        type: string
+      test_config_from_source:
+        description: 'DEBUG: Use config.json from current source branch instead of dev'
+        required: false
+        default: false
+        type: boolean
   workflow_dispatch:
     inputs:
       test_level:
@@ -31,12 +42,12 @@ on:
         default: false
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.event_name == 'workflow_call' && format('{0}-{1}-{2}', github.workflow, github.ref, github.run_id) || format('{0}-{1}', github.workflow, github.ref) }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
 
 jobs:
   build_and_test:
-    name: Build and Test on GPU (${{ matrix.runner }})
+    name: Build and Test on GPU (${{ matrix.runner }}) - Level ${{ inputs.test_level || '1' }}
     timeout-minutes: 720
     runs-on: ${{ matrix.runner }}
     strategy: