Add support for running Data Prep using Airflow

.github/workflows/auth-ci.yml

@@ -26,30 +26,42 @@ jobs:
       with:
         python-version: '3.11'
 
-    - name: Install dependencies
+    - name: Install dependencies - Client
       working-directory: .
       run: |
         python -m pip install --upgrade pip
         pip install -e cli/
         pip install -r cli/test-requirements.txt
-        pip install -r server/requirements.txt
-        pip install -r server/test-requirements.txt
+
+    - name: Install Dependencies - Server
+      working-directory: ./server
+      run: |
+        python -m venv .venv_server
+        source .venv_server/bin/activate
+        pip install -r requirements.txt
+        pip install -r test-requirements.txt
 
     - name: Set server environment vars
       working-directory: ./server
       run: cp .env.local.online-auth .env
 
     - name: Run postgresql server in background
       working-directory: ./server
-      run: sh run_dev_postgresql.sh && sleep 6
+      run: |
+        python -m venv .venv_server
+        sh run_dev_postgresql.sh && sleep 6
 
     - name: Run django server in background with generated certs
       working-directory: ./server
-      run: sh setup-dev-server.sh & sleep 6
+      run: |
+        python -m venv .venv_server
+        sh setup-dev-server.sh & sleep 6
 
     - name: Run server integration tests
       working-directory: ./server
-      run: python seed.py --cert cert.crt --auth online
+      run: |
+        python -m venv .venv_server
+        python seed.py --cert cert.crt --auth online
 
     - name: Run client integration tests
       working-directory: .

.github/workflows/docker-ci.yml

@@ -34,22 +34,31 @@ jobs:
         make
         sudo make install
 
-    - name: Install dependencies
+    - name: Install dependencies - Client
       working-directory: .
       run: |
         python -m pip install --upgrade pip
         pip install -e cli/
         pip install -r cli/test-requirements.txt
-        pip install -r server/requirements.txt
-        pip install -r server/test-requirements.txt
+
+    - name: Install Dependencies - Server
+      working-directory: ./server
+      run: |
+        python -m venv .venv_server
+        source .venv_server/bin/activate
+        pip install -r requirements.txt
+        pip install -r test-requirements.txt
+
 
     - name: Set server environment vars
       working-directory: ./server
       run: cp .env.local.local-auth.sqlite .env
 
     - name: Generate SSL certificate
       working-directory: ./server
-      run: sh setup-dev-server.sh -c cert.crt -k cert.key -d 0
+      run: |
+        python -m venv .venv_server
+        sh setup-dev-server.sh -c cert.crt -k cert.key -d 0
 
     - name: Build container image
       working-directory: ./server

.github/workflows/encrypted-containers-ci.yml

@@ -34,30 +34,42 @@ jobs:
           make
           sudo make install
 
-      - name: Install dependencies
+      - name: Install dependencies - Client
         working-directory: .
         run: |
           python -m pip install --upgrade pip
           pip install -e cli/
           pip install -r cli/test-requirements.txt
-          pip install -r server/requirements.txt
-          pip install -r server/test-requirements.txt
+
+      - name: Install Dependencies - Server
+        working-directory: ./server
+        run: |
+          python -m venv .venv_server
+          source .venv_server/bin/activate
+          pip install -r requirements.txt
+          pip install -r test-requirements.txt
 
       - name: Set server environment vars
         working-directory: ./server
         run: cp .env.local.local-auth .env
 
       - name: Run postgresql server in background
         working-directory: ./server
-        run: sh run_dev_postgresql.sh && sleep 6
+        run: |
+          source .venv_server/bin/activate
+          sh run_dev_postgresql.sh && sleep 6
 
       - name: Run django server in background with generated certs
         working-directory: ./server
-        run: sh setup-dev-server.sh & sleep 6
+        run: |
+          source .venv_server/bin/activate
+          sh setup-dev-server.sh & sleep 6
 
       - name: Run server integration tests
         working-directory: ./server
-        run: python seed.py --cert cert.crt
+        run: |
+          source .venv_server/bin/activate
+          python seed.py --cert cert.crt
 
       - name: Run client integration tests
         working-directory: .

.github/workflows/local-ci.yml

@@ -34,32 +34,61 @@ jobs:
         make
         sudo make install
 
-    - name: Install dependencies
+    - name: Install dependencies - Client
       working-directory: .
       run: |
         python -m pip install --upgrade pip
         pip install -e cli/
         pip install -r cli/test-requirements.txt
-        pip install -r server/requirements.txt
-        pip install -r server/test-requirements.txt
+
+    - name: Install Dependencies - Server
+      working-directory: ./server
+      run: |
+        python -m venv .venv_server
+        source .venv_server/bin/activate
+        pip install -r requirements.txt
+        pip install -r test-requirements.txt
 
     - name: Set server environment vars
       working-directory: ./server
       run: cp .env.local.local-auth .env
 
     - name: Run postgresql server in background
       working-directory: ./server
-      run: sh run_dev_postgresql.sh && sleep 6
+      run: |
+        source .venv_server/bin/activate
+        sh run_dev_postgresql.sh && sleep 6
 
     - name: Run django server in background with generated certs
       working-directory: ./server
-      run: sh setup-dev-server.sh & sleep 6
+      run: |
+        source .venv_server/bin/activate
+        sh setup-dev-server.sh & sleep 6
 
     - name: Run server integration tests
       working-directory: ./server
-      run: python seed.py --cert cert.crt
+      run: |
+        source .venv_server/bin/activate
+        python seed.py --cert cert.crt
+
+    - name: Run chestxray demo including private model
+      working-directory: .
+      run: sh cli/cli_chestxray_tutorial_test.sh -f -p
+
+    - name: Reset DB for Workflow test
+      working-directory: ./server
+      run: |
+        source .venv_server/bin/activate
+        sh reset_db.sh
+        sh reset_db_postgresql.sh
+
+    - name: Seed DB for Workflow test
+      working-directory: ./server
+      run: |
+        source .venv_server/bin/activate
+        python seed.py --cert cert.crt -w
 
-    - name: Run chestxray demo
+    - name: Run chestxray demo with workflow (no private model)
       working-directory: .
       run: sh cli/cli_chestxray_tutorial_test.sh -f
 

.github/workflows/train-ci.yml

@@ -15,30 +15,42 @@ jobs:
       with:
         python-version: '3.11'
 
-    - name: Install dependencies
+    - name: Install dependencies - Client
       working-directory: .
       run: |
         python -m pip install --upgrade pip
         pip install -e cli/
         pip install -r cli/test-requirements.txt
-        pip install -r server/requirements.txt
-        pip install -r server/test-requirements.txt
+
+    - name: Install Dependencies - Server
+      working-directory: ./server
+      run: |
+        python -m venv .venv_server
+        source .venv_server/bin/activate
+        pip install -r requirements.txt
+        pip install -r test-requirements.txt
 
     - name: Set server environment vars
       working-directory: ./server
       run: cp .env.local.local-auth .env
 
     - name: Run postgresql server in background
       working-directory: ./server
-      run: sh run_dev_postgresql.sh && sleep 6
+      run: |
+        source .venv_server/bin/activate
+        sh run_dev_postgresql.sh && sleep 6
 
     - name: Run django server in background with generated certs
       working-directory: ./server
-      run: sh setup-dev-server.sh & sleep 6
+      run: |
+        source .venv_server/bin/activate
+        sh setup-dev-server.sh & sleep 6
 
     - name: Run server integration tests
       working-directory: ./server
-      run: python seed.py --cert cert.crt
+      run: |
+        source .venv_server/bin/activate
+        python seed.py --cert cert.crt
 
     - name: Run client integration tests
       working-directory: .

.github/workflows/unittests.yml

@@ -16,14 +16,23 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.11
-    - name: Install dependencies
+
+    - name: Install dependencies - Client
+      working-directory: .
       run: |
         python -m pip install --upgrade pip
         pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         if [ -f cli/requirements.txt ]; then pip install -e cli; fi
-        pip install -r server/requirements.txt
-        pip install -r server/test-requirements.txt
+
+    - name: Install Dependencies - Server
+      working-directory: ./server
+      run: |
+        python -m venv .venv_server
+        source .venv_server/bin/activate
+        pip install -r requirements.txt
+        pip install -r test-requirements.txt
+
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -34,20 +43,29 @@ jobs:
         # Exclude examples folder as it doesn't contain code related to medperf tools
         # Exclude migrations folder as it contains autogenerated code
         # Ignore E231, as it is raising warnings with auto-generated code.
-        flake8 . --count --max-complexity=10 --max-line-length=127 --ignore F821,W503,E231 --statistics --exclude=examples/,"*/migrations/*",cli/medperf/templates/
+        flake8 . --count --max-complexity=10 --max-line-length=127 --ignore F821,W503,E231 --statistics --exclude=examples/,"*/migrations/*",cli/medperf/templates/,server/.venv_server/
+
     - name: Test with pytest
       working-directory: ./cli/medperf/tests
       run: |
         pytest
+
     - name: Set server environment vars
       working-directory: ./server
       run: cp .env.local.local-auth .env
+
     - name: Run postgresql server in background
       working-directory: ./server
       run: sh run_dev_postgresql.sh && sleep 6
+
     - name: Run migrations
       working-directory: ./server
-      run: python manage.py migrate
+      run: |
+        source .venv_server/bin/activate
+        python manage.py migrate
+
     - name: Run server unit tests
       working-directory: ./server
-      run: python manage.py test --parallel
+      run: |
+        source .venv_server/bin/activate
+        python manage.py test --parallel

.github/workflows/webui-ci.yml

@@ -15,30 +15,42 @@ jobs:
       with:
         python-version: '3.11'
 
-    - name: Install dependencies
+    - name: Install dependencies - Client
       working-directory: .
       run: |
         python -m pip install --upgrade pip
         pip install -e cli/
         pip install -r cli/test-requirements.txt
-        pip install -r server/requirements.txt
-        pip install -r server/test-requirements.txt
+
+    - name: Install Dependencies - Server
+      working-directory: ./server
+      run: |
+        python -m venv .venv_server
+        source .venv_server/bin/activate
+        pip install -r requirements.txt
+        pip install -r test-requirements.txt
 
     - name: Set server environment vars
       working-directory: ./server
       run: cp .env.local.local-auth .env
 
     - name: Run postgresql server in background
       working-directory: ./server
-      run: sh run_dev_postgresql.sh && sleep 6
+      run: |
+        source .venv_server/bin/activate
+        sh run_dev_postgresql.sh && sleep 6
 
     - name: Run django server in background with generated certs
       working-directory: ./server
-      run: sh setup-dev-server.sh & sleep 6
+      run: |
+        source .venv_server/bin/activate
+        sh setup-dev-server.sh & sleep 6
 
     - name: Reset and seed database with tutorial data
       working-directory: ./server
-      run: sh reset_db.sh && python seed.py --demo tutorial
+      run: |
+        source .venv_server/bin/activate
+        sh reset_db.sh && python seed.py --demo tutorial
 
     - name: Set up tutorial files
       working-directory: .