add a workflow to test database

.github/workflows/test_database.yml

@@ -0,0 +1,183 @@
+name: Test Database
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test_targetdb
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up tbls
+        uses: k1low/setup-tbls@v1
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Cache pip packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .
+          pip install pytest pytest-cov alembic
+
+      - name: Create test config
+        run: |
+          mkdir -p ~/.pfs
+          cat > ~/.pfs/test_db_config.toml << EOF
+          [targetdb.db]
+          dialect = "postgresql"
+          user = "postgres"
+          password = "postgres"
+          host = "localhost"
+          port = 5432
+          dbname = "test_targetdb"
+
+          [uploader]
+          host = "localhost"
+          data_dir = "examples/data/uploader"
+          EOF
+
+      - name: Create database
+        run: |
+          pfs-targetdb-cli create-db --config ~/.pfs/test_db_config.toml
+
+      - name: Setup PostgreSQL with q3c
+        run: |
+          # Identify the PostgreSQL container
+          POSTGRES_CONTAINER_ID=$(docker ps -q --filter "ancestor=postgres:16")
+          echo "PostgreSQL container ID: $POSTGRES_CONTAINER_ID"
+
+          # Install necessary packages inside the PostgreSQL container
+          docker exec $POSTGRES_CONTAINER_ID apt-get update
+          docker exec $POSTGRES_CONTAINER_ID apt-get install -y \
+            postgresql-server-dev-16 \
+            build-essential \
+            git \
+            libreadline-dev \
+            zlib1g-dev \
+            libzstd-dev \
+            liblz4-dev \
+            libssl-dev
+
+          # Clone q3c repository inside the container
+          docker exec $POSTGRES_CONTAINER_ID bash -c "mkdir -p /build && cd /build && git clone https://github.com/segasai/q3c.git"
+
+          # Build and install q3c inside the container
+          docker exec $POSTGRES_CONTAINER_ID bash -c "cd /build/q3c && sed -i 's/LLVM ?= yes/LLVM = no/g' Makefile && make && make install"
+
+          # Verify installation
+          echo "Verifying q3c installation in PostgreSQL container:"
+          docker exec $POSTGRES_CONTAINER_ID bash -c "ls -la \$(pg_config --pkglibdir)/q3c.so"
+          docker exec $POSTGRES_CONTAINER_ID bash -c "ls -la \$(pg_config --sharedir)/extension/q3c*"
+
+      - name: Install q3c extension
+        run: |
+          pfs-targetdb-cli install-q3c --config ~/.pfs/test_db_config.toml
+
+      - name: Create schema
+        run: |
+          pfs-targetdb-cli create-schema --config ~/.pfs/test_db_config.toml
+
+      - name: Insert example data
+        run: |
+          pfs-targetdb-cli insert -c ~/.pfs/test_db_config.toml -t target_type examples/data/target_types.csv --commit
+          pfs-targetdb-cli insert -c ~/.pfs/test_db_config.toml -t filter_name examples/data/filter_names.csv --commit
+          pfs-targetdb-cli insert -c ~/.pfs/test_db_config.toml -t partner examples/data/partner.csv --commit
+          pfs-targetdb-cli insert -c ~/.pfs/test_db_config.toml -t pfs_arm examples/data/pfs_arm.csv --commit
+          pfs-targetdb-cli insert -c ~/.pfs/test_db_config.toml -t proposal_category examples/data/proposal_category.csv --commit
+          pfs-targetdb-cli insert -c ~/.pfs/test_db_config.toml -t proposal examples/data/proposals.csv --commit
+          pfs-targetdb-cli insert -c ~/.pfs/test_db_config.toml -t input_catalog examples/data/input_catalogs.csv --commit
+
+      - name: Insert input catalog for fluxstd
+        run: |
+          pfs-targetdb-cli insert -c ~/.pfs/test_db_config.toml -t input_catalog examples/data/input_catalog_fluxstd.csv --commit
+
+      - name: Prepare fluxstd file"
+        run: |
+          pfs-targetdb-cli prep-fluxstd examples/data/fluxstd/Fstar_v3.3/feather-original examples/data/fluxstd/Fstar_v3.3/feather \
+            --version "3.3" \
+            --input_catalog_id 3006 \
+            --rename-cols '{"fstar_gaia": "is_fstar_gaia"}' \
+            --format feather
+
+      - name: Insert fluxstd data
+        run: |
+          pfs-targetdb-cli insert -c ~/.pfs/test_db_config.toml \
+            -t fluxstd \
+            examples/data/fluxstd/Fstar_v3.3/feather/ra354.8_354.9_dec-40.0_90.0.feather \
+            --commit
+
+      - name: Insert proposals and input catalogs for science target
+        run: |
+          pfs-targetdb-cli insert -c ~/.pfs/test_db_config.toml examples/data/targets/example_proposals.csv -t proposal --commit
+          pfs-targetdb-cli insert -c ~/.pfs/test_db_config.toml examples/data/targets/example_input_catalogs.csv -t input_catalog --commit --fetch
+
+      - name: Transfer target lists
+        run: |
+          pfs-targetdb-cli transfer-targets -c ~/.pfs/test_db_config.toml examples/data/targets/example_input_catalogs.csv \
+            --local-dir examples/data/targets --force
+
+      - name: Insert targets
+        run: |
+          pfs-targetdb-cli insert-targets -c ~/.pfs/test_db_config.toml examples/data/targets/example_input_catalogs.csv \
+            --data-dir examples/data/targets/ --commit
+
+      - name: Insert custom pointings
+        run: |
+          pfs-targetdb-cli insert-pointings -c ~/.pfs/test_db_config.toml examples/data/targets/example_input_catalogs.csv \
+            --data-dir examples/data/targets/ --commit
+
+      - name: Change input catalog active status
+        run: |
+          pfs-targetdb-cli update-catalog-active 1004 true -c ~/.pfs/test_db_config.toml --verbose --commit
+
+      - name: Generate ER diagram
+        run: |
+          pfs-targetdb-cli diagram -c ~/.pfs/test_db_config.toml --generator tbls --output-dir ./diagram-test
+
+      #   - name: Run tests
+      #     run: |
+      #       pytest -xvs tests/ --cov=src/targetdb/ --cov-report=xml
+
+      #   - name: Upload coverage to Codecov
+      #     uses: codecov/codecov-action@v4
+      #     with:
+      #       token: ${{ secrets.CODECOV_TOKEN }}
+      #       file: ./coverage.xml
+      #       fail_ci_if_error: false
+
+    #   - name: Test migrations
+    #     run: |
+    #       alembic -c alembic/local_test/alembic/alembic.ini upgrade head
+    #       alembic -c alembic/local_test/alembic/alembic.ini downgrade base

examples/data/input_catalog_fluxstd.csv

@@ -0,0 +1,8 @@
+input_catalog_id,input_catalog_name,input_catalog_description,upload_id
+3000,"Fstar_v1.0","F-type star candidates version 1.0 for flux calibration",
+3001,"Fstar_v2.0","F-type star candidates version 2.0 for flux calibration",
+3002,"Fstar_v2.1","F-type star candidates version 2.1 for flux calibration",
+3003,"Fstar_v3.0","F-type star candidates version 3.0 for flux calibration",
+3004,"Fstar_v3.1","F-type star candidates version 3.1 for flux calibration",
+3005,"Fstar_v3.2","F-type star candidates version 3.2 for flux calibration",
+3006,"Fstar_v3.3","F-type star candidates version 3.3 for flux calibration",

examples/data/input_catalogs.csv

@@ -9,7 +9,7 @@ input_catalog_id,input_catalog_name,input_catalog_description,upload_id
 7,scssp_pdr2_wide,HSC-SSP Public Data Release 2 (Wide),
 8,hscssp_pdr2_dud,HSC-SSP Public Dta Release 2 (Deep+UltraDeep),
 9,hscssp_pdr3_wide,HSC-SSP Public Data Release 3 (Wide),
-0,hscssp_pdr3_dud,HSC-SSP Public Data Release 3 (Deep+UltraDeep),
+10,hscssp_pdr3_dud,HSC-SSP Public Data Release 3 (Deep+UltraDeep),
 11,hscssp_pdr4_wide,HS-SSP Public Data Release 4 (Wide),
 12,hscssp_pdr4_dud,HSC-SSP Public Data Release 4 (Dee+UltraDeep),
 1001,sky_hscssp_s21a_wide,Sky positions from S21A HSC-SSP (Wide),

examples/data/proposals.csv

@@ -1,38 +1,38 @@
-proposal_id,group_id,pi_first_name,pi_last_name,pi_middle_name,rank,grade,allocated_time_total,allocated_time_lr,allocated_time_mr,proposal_category_id
-S21B-EN01,o21001,Hirofumi,Okita,None,9.0,a,240000.0,,,1
-S21B-EN16,o21016,Shintaro,Koshida,None,9.0,a,240000.0,,,1
-S21B-EN99,o21999,Masato,Onodera,None,1.0,a,1.0,,,1
-S21B-QN078,o21628,Takashi,Moriya,None,9.5,a,120000.0,,,1
-S21B-TE037-KQ,o21150,George,Becker,None,9.5,a,60000.0,,,2
-S21B-TE114-GQ,o21154,David,Sand,None,9.5,a,12000.0,,,3
-S21B-UH006-A1Q,o21165,Lukas,Zalesky,M.,9.5,a,4800.0,,,4
-S22A-EN16,o22016,Shintaro,Koshida,None,9.0,a,240000.0,,,1
-S22B-EN16,o22016,Shintaro,Koshida,None,9.0,a,240000.0,,,1
-S22B-EN16-CO,o22016,Kiyoto,Yabe,None,9.0,a,240000.0,,,1
-S23A-EN16,o22016,Shintaro,Koshida,None,9.0,a,240000.0,,,1
-S23A-QN900,o22016,Kiyoto,Yabe0,None,6.0,c,36000.0,,,1
-S23A-QN901,o22016,Kiyoto,Yabe1,None,7.0,b,12000.0,,,1
-S23A-QN902,o22016,Kiyoto,Yabe2,None,8.0,b,7200.0,,,1
-S23A-QN903,o22016,Kiyoto,Yabe3,None,9.0,a,3600.0,,,1
-S23B-QT901,,M.,Ishigaki,,10.0,A,595.5,319.5,276.0,1
-S23B-QT902,,K.,Yabe,,8.8,A,1730.0,1730.0,0.0,1
-S23B-QT903,,W.,He,,9.1,B,234.5,0.0,234.5,1
-S23B-QT904,,M.,Tanaka,,8.2,B,62.25,0.0,62.25,1
-S23B-QT905,,M.,Onodera,,7.5,B,5166.0,0.0,5166.0,1
-S23B-QT906,,Y.,Moritani,,7.0,A,2542.25,2542.25,0.0,1
-S23B-QT907,,J.,Eric,,6.5,C,2594.5,2594.5,0.0,1
-S23B-QT908,,Y.,Takagi,,5.6,C,320.0,320.0,0.0,1
-S24B-QT901,o00001,Sakurako,Okamoto,NaN,9.0,A,284.25,,284.25,1
-S24B-QT902,o00002,Masayuki,Tanaka,NaN,8.5,B,951.0,,951.0,1
-S24B-QT903,o00003,Tae-Soo,Pyo,NaN,8.5,B,3190.75,,3190.75,1
-S24B-QT904,o00004,Akira,Arai,NaN,6.0,B,409.0,,409.0,1
-S24B-QT905,o00005,Yuhei,Takagi,NaN,7.0,B,532.75,,532.75,1
-S24B-QT906,o00006,Miho,Ishigaki,NaN,5.5,C,0.0,,,1
-S24B-QT907,o00007,Naoyuki,Tamura,NaN,6.5,B,8140.5,8140.5,,1
-S24B-QT908,o00008,Kiyoto,Yabe,NaN,8.0,B,707.75,707.75,,1
-S24B-QT909,o00009,Eric,Jeschke,NaN,7.0,B,2567.75,2567.75,,1
-S24B-QT910,o00010,Wanqiu,He,NaN,7.5,B,5307.25,5307.25,,1
-S24B-QT911,o00011,Mao,Zhiying,NaN,10.0,A,5348.05,5348.05,,1
-S24B-QT912,o00012,Yuki,Moritani,NaN,9.5,A,350.25,350.25,,1
-S24B-QT913,o00013,Vera,Passegger,Maria,7.0,B,263.0,263.0,,1
-S24B-QT914,o00014,Miho,Ishigaki,NaN,0.0,F,0.0,0.0,0.0,1
+proposal_id,group_id,pi_first_name,pi_last_name,pi_middle_name,rank,grade,allocated_time_total,allocated_time_lr,allocated_time_mr,proposal_category_id,partner_name,is_too
+S21B-EN01,o21001,Hirofumi,Okita,None,9.0,a,240000.0,0.0,0.0,1,subaru,False
+S21B-EN16,o21016,Shintaro,Koshida,None,9.0,a,240000.0,0.0,0.0,1,subaru,False
+S21B-EN99,o21999,Masato,Onodera,None,1.0,a,1.0,0.0,0.0,1,subaru,False
+S21B-QN078,o21628,Takashi,Moriya,None,9.5,a,120000.0,0.0,0.0,1,subaru,False
+S21B-TE037-KQ,o21150,George,Becker,None,9.5,a,60000.0,0.0,0.0,1,subaru,False
+S21B-TE114-GQ,o21154,David,Sand,None,9.5,a,12000.0,0.0,0.0,1,subaru,False
+S21B-UH006-A1Q,o21165,Lukas,Zalesky,M.,9.5,a,4800.0,0.0,0.0,1,subaru,False
+S22A-EN16,o22016,Shintaro,Koshida,None,9.0,a,240000.0,0.0,0.0,1,subaru,False
+S22B-EN16,o22016,Shintaro,Koshida,None,9.0,a,240000.0,0.0,0.0,1,subaru,False
+S22B-EN16-CO,o22016,Kiyoto,Yabe,None,9.0,a,240000.0,0.0,0.0,1,subaru,False
+S23A-EN16,o22016,Shintaro,Koshida,None,9.0,a,240000.0,0.0,0.0,1,subaru,False
+S23A-QN900,o22016,Kiyoto,Yabe0,None,6.0,c,36000.0,0.0,0.0,1,subaru,False
+S23A-QN901,o22016,Kiyoto,Yabe1,None,7.0,b,12000.0,0.0,0.0,1,subaru,False
+S23A-QN902,o22016,Kiyoto,Yabe2,None,8.0,b,7200.0,0.0,0.0,1,subaru,False
+S23A-QN903,o22016,Kiyoto,Yabe3,None,9.0,a,3600.0,0.0,0.0,1,subaru,False
+S23B-QT901,,M.,Ishigaki,,10.0,A,595.5,319.5,276.0,1,subaru,False
+S23B-QT902,,K.,Yabe,,8.8,A,1730.0,1730.0,0.0,1,subaru,False
+S23B-QT903,,W.,He,,9.1,B,234.5,0.0,234.5,1,subaru,False
+S23B-QT904,,M.,Tanaka,,8.2,B,62.25,0.0,62.25,1,subaru,False
+S23B-QT905,,M.,Onodera,,7.5,B,5166.0,0.0,5166.0,1,subaru,False
+S23B-QT906,,Y.,Moritani,,7.0,A,2542.25,2542.25,0.0,1,subaru,False
+S23B-QT907,,J.,Eric,,6.5,C,2594.5,2594.5,0.0,1,subaru,False
+S23B-QT908,,Y.,Takagi,,5.6,C,320.0,320.0,0.0,1,subaru,False
+S24B-QT901,o00001,Sakurako,Okamoto,NaN,9.0,A,284.25,0.0,284.25,1,subaru,False
+S24B-QT902,o00002,Masayuki,Tanaka,NaN,8.5,B,951.0,0.0,951.0,1,subaru,False
+S24B-QT903,o00003,Tae-Soo,Pyo,NaN,8.5,B,3190.75,0.0,3190.75,1,subaru,False
+S24B-QT904,o00004,Akira,Arai,NaN,6.0,B,409.0,0.0,409.0,1,subaru,False
+S24B-QT905,o00005,Yuhei,Takagi,NaN,7.0,B,532.75,0.0,532.75,1,subaru,False
+S24B-QT906,o00006,Miho,Ishigaki,NaN,5.5,C,0.0,0.0,0.0,1,subaru,False
+S24B-QT907,o00007,Naoyuki,Tamura,NaN,6.5,B,8140.5,8140.5,0.0,1,subaru,False
+S24B-QT908,o00008,Kiyoto,Yabe,NaN,8.0,B,707.75,707.75,0.0,1,subaru,False
+S24B-QT909,o00009,Eric,Jeschke,NaN,7.0,B,2567.75,2567.75,0.0,1,subaru,False
+S24B-QT910,o00010,Wanqiu,He,NaN,7.5,B,5307.25,5307.25,0.0,1,subaru,False
+S24B-QT911,o00011,Mao,Zhiying,NaN,10.0,A,5348.05,5348.05,0.0,1,subaru,False
+S24B-QT912,o00012,Yuki,Moritani,NaN,9.5,A,350.25,350.25,0.0,1,subaru,False
+S24B-QT913,o00013,Vera,Passegger,Maria,7.0,B,263.0,263.0,0.0,1,subaru,False
+S24B-QT914,o00014,Miho,Ishigaki,NaN,0.0,F,0.0,0.0,0.0,1,subaru,False

examples/data/targets/20240921-041833-c9d7ad93118243b5/README.txt

@@ -0,0 +1,15 @@
+# README for output files from the online PFS pointing planner (PPP)
+
+A ZIP file generated by the online PPP in the PFS target uploader contains the following files.
+
+- README.txt: This file
+- target*.ecsv: cleaned target list (ECSV)
+- target_summary*.ecsv: summary table of input targets grouped by priority and resolution (ECSV)
+- psl*.ecsv: summary of the online PPP simulation including requested observing time, completion rate, etc. (ECSV)
+- ppc*.ecsv: list of PFS pointings derived by the online PPP simulation sorted by pointing priority and grouped by resolution (ECSV)
+- ppp_figure*.html: standalone plots shown as the result of the online PPP simulation (HTML)
+- <original target list>: original input target list
+
+About the PFS target uploader, visit https://pfs-etc.naoj.hawaii.edu/uploader/app and User Guide (https://pfs-etc.naoj.hawaii.edu/uploader/doc/index.html).
+
+About the Enhanced Character-Separated Values (ECSV) format, visit https://docs.astropy.org/en/stable/io/ascii/ecsv.html