From 7e4091f973bca7c2a4e84c5ad3ddf0f24e090877 Mon Sep 17 00:00:00 2001 From: LucieContamin Date: Wed, 7 May 2025 11:24:13 -0400 Subject: [PATCH 1/5] add gh actions --- .github/workflows/R-CMD-check.yaml | 60 ++++++++++++++++++++++++++++ .github/workflows/lint.yaml | 34 ++++++++++++++++ .github/workflows/test-coverage.yaml | 53 ++++++++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 .github/workflows/R-CMD-check.yaml create mode 100644 .github/workflows/lint.yaml create mode 100644 .github/workflows/test-coverage.yaml diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml new file mode 100644 index 0000000..ac17169 --- /dev/null +++ b/.github/workflows/R-CMD-check.yaml @@ -0,0 +1,60 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: main + pull_request: + branches: main + workflow_dispatch: + schedule: + - cron: "13 9 15 * *" + +name: R-CMD-check +permissions: + contents: read + +jobs: + R-CMD-check: + runs-on: ${{ matrix.config.os }} + + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + + strategy: + fail-fast: false + matrix: + config: + - {os: macos-latest, r: 'release'} + - {os: windows-latest, r: 'release'} + - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} + - {os: ubuntu-latest, r: 'release'} + - {os: ubuntu-latest, r: 'oldrel-1'} + + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + R_KEEP_PKG_SOURCE: yes + + steps: + - id: checkout + uses: actions/checkout@v4 + + - id: setup-pandoc + uses: r-lib/actions/setup-pandoc@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 + + - id: setup-r + uses: r-lib/actions/setup-r@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 + with: + r-version: ${{ matrix.config.r }} + http-user-agent: ${{ matrix.config.http-user-agent }} + use-public-rspm: true + extra-repositories: https://hubverse-org.r-universe.dev + + - id: fetch-dependencies + uses: r-lib/actions/setup-r-dependencies@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 + with: + extra-packages: any::rcmdcheck + needs: check + + - id: check + uses: r-lib/actions/check-r-package@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 + with: + upload-snapshots: true diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000..7ce3fd6 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,34 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: main + pull_request: + branches: main + +name: lint +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-r@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 + with: + extra-packages: any::lintr, local::. + needs: lint + + - name: Lint + run: lintr::lint_package() + shell: Rscript {0} + env: + LINTR_ERROR_ON_LINT: true diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml new file mode 100644 index 0000000..2062931 --- /dev/null +++ b/.github/workflows/test-coverage.yaml @@ -0,0 +1,53 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: main + pull_request: + branches: main + +name: test-coverage +permissions: + contents: read + +jobs: + test-coverage: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-r@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 + with: + use-public-rspm: true + extra-repositories: https://hubverse-org.r-universe.dev + + - uses: r-lib/actions/setup-r-dependencies@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 + with: + extra-packages: any::covr + needs: coverage + + - name: Test coverage + run: | + covr::codecov( + quiet = FALSE, + clean = FALSE, + install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package") + ) + shell: Rscript {0} + + - name: Show testthat output + if: always() + run: | + ## -------------------------------------------------------------------- + find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true + shell: bash + + - name: Upload test results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: coverage-test-failures + path: ${{ runner.temp }}/package From 8857850283661189e8fc1d9d821caa48428c9e46 Mon Sep 17 00:00:00 2001 From: LucieContamin Date: Wed, 7 May 2025 11:38:52 -0400 Subject: [PATCH 2/5] update permissions --- .github/workflows/test-coverage.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 2062931..2b19029 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -7,8 +7,6 @@ on: branches: main name: test-coverage -permissions: - contents: read jobs: test-coverage: From 879f512bce18ab7e7145384faa38cdbb106f776d Mon Sep 17 00:00:00 2001 From: LucieContamin Date: Wed, 7 May 2025 12:00:11 -0400 Subject: [PATCH 3/5] update gh actions --- .github/workflows/R-CMD-check.yaml | 2 -- .github/workflows/test-coverage.yaml | 51 ---------------------------- 2 files changed, 53 deletions(-) delete mode 100644 .github/workflows/test-coverage.yaml diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index ac17169..a49abbe 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -10,8 +10,6 @@ on: - cron: "13 9 15 * *" name: R-CMD-check -permissions: - contents: read jobs: R-CMD-check: diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml deleted file mode 100644 index 2b19029..0000000 --- a/.github/workflows/test-coverage.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples -# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help -on: - push: - branches: main - pull_request: - branches: main - -name: test-coverage - -jobs: - test-coverage: - runs-on: ubuntu-latest - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - - steps: - - uses: actions/checkout@v4 - - - uses: r-lib/actions/setup-r@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 - with: - use-public-rspm: true - extra-repositories: https://hubverse-org.r-universe.dev - - - uses: r-lib/actions/setup-r-dependencies@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 - with: - extra-packages: any::covr - needs: coverage - - - name: Test coverage - run: | - covr::codecov( - quiet = FALSE, - clean = FALSE, - install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package") - ) - shell: Rscript {0} - - - name: Show testthat output - if: always() - run: | - ## -------------------------------------------------------------------- - find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true - shell: bash - - - name: Upload test results - if: failure() - uses: actions/upload-artifact@v4 - with: - name: coverage-test-failures - path: ${{ runner.temp }}/package From 258755ad97b9ba7897218747e6a502939c1cb3af Mon Sep 17 00:00:00 2001 From: LucieContamin Date: Wed, 7 May 2025 15:12:42 -0400 Subject: [PATCH 4/5] update test workflow --- tests/testthat/test_submission.R | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/testthat/test_submission.R b/tests/testthat/test_submission.R index 95a08dd..c70345d 100644 --- a/tests/testthat/test_submission.R +++ b/tests/testthat/test_submission.R @@ -1,8 +1,18 @@ test_that("Test validation process", { - obs <- "../exp/target-data/time-series.csv" - hub_path <- "../exp/" - pop_path <- "../exp/auxiliary-data/location_census/locations.csv" + hub_path0 <- tempdir() + hub_path <- paste0(hub_path0, "/exp/") + all_dir <- c(hub_path, + grep("\\.", paste0(hub_path, dir("../exp", recursive = TRUE, + include.dirs = TRUE)), + value = TRUE, invert = TRUE)) + purrr::map(all_dir, dir.create, showWarnings = FALSE) + file.copy(dir("../exp", full.names = TRUE, recursive = TRUE), + gsub("^../exp", hub_path, dir("../exp", full.names = TRUE, + recursive = TRUE))) + + obs <- paste0(hub_path, "target-data/time-series.csv") + pop_path <- paste0(hub_path, "auxiliary-data/location_census/locations.csv") merge_sample_col <- c("run_grouping", "stochastic_run") partition <- round_id <- r_schema <- NULL n_decimal <- 1 @@ -463,4 +473,7 @@ test_that("Test validation process", { merge_sample_col = merge_sample_col)) expect_contains(attr(check$result$spl_compound_tid, "class"), c("error", "check_error")) + + unlink(hub_path0, recursive = TRUE) + }) From c427b4c9569d9b9719c67c27ba5775ab0bd5b7f2 Mon Sep 17 00:00:00 2001 From: LucieContamin Date: Wed, 7 May 2025 16:00:18 -0400 Subject: [PATCH 5/5] rm cmd check + minor fix (gc) - temp --- .github/workflows/R-CMD-check.yaml | 58 ------------------------------ R/imports-hubValidations.R | 4 +-- R/validate_submission.R | 10 +++++- 3 files changed, 11 insertions(+), 61 deletions(-) delete mode 100644 .github/workflows/R-CMD-check.yaml diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml deleted file mode 100644 index a49abbe..0000000 --- a/.github/workflows/R-CMD-check.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples -# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help -on: - push: - branches: main - pull_request: - branches: main - workflow_dispatch: - schedule: - - cron: "13 9 15 * *" - -name: R-CMD-check - -jobs: - R-CMD-check: - runs-on: ${{ matrix.config.os }} - - name: ${{ matrix.config.os }} (${{ matrix.config.r }}) - - strategy: - fail-fast: false - matrix: - config: - - {os: macos-latest, r: 'release'} - - {os: windows-latest, r: 'release'} - - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} - - {os: ubuntu-latest, r: 'release'} - - {os: ubuntu-latest, r: 'oldrel-1'} - - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - R_KEEP_PKG_SOURCE: yes - - steps: - - id: checkout - uses: actions/checkout@v4 - - - id: setup-pandoc - uses: r-lib/actions/setup-pandoc@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 - - - id: setup-r - uses: r-lib/actions/setup-r@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 - with: - r-version: ${{ matrix.config.r }} - http-user-agent: ${{ matrix.config.http-user-agent }} - use-public-rspm: true - extra-repositories: https://hubverse-org.r-universe.dev - - - id: fetch-dependencies - uses: r-lib/actions/setup-r-dependencies@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 - with: - extra-packages: any::rcmdcheck - needs: check - - - id: check - uses: r-lib/actions/check-r-package@bd49c52ffe281809afa6f0fecbf37483c5dd0b93 #v2.11.3 - with: - upload-snapshots: true diff --git a/R/imports-hubValidations.R b/R/imports-hubValidations.R index 278ff6c..3d17bc6 100644 --- a/R/imports-hubValidations.R +++ b/R/imports-hubValidations.R @@ -108,8 +108,8 @@ check_tbl_rows_unique <- function(tbl, file_path, hub_path) { #' @importFrom purrr imap check_tbl_value_col <- function(tbl, round_id, file_path, hub_path) { config_tasks <- read_config(hub_path, "tasks") - tbl <- dplyr::mutate(tbl, dplyr::across(-dplyr::contains("value"), - as.character)) + tbl <- dplyr::mutate(tbl, + dplyr::across(-dplyr::contains("value"), as.character)) details <- split(tbl, f = tbl$output_type) |> purrr::imap( \(.x, .y) { diff --git a/R/validate_submission.R b/R/validate_submission.R index 4b3a04a..78061db 100644 --- a/R/validate_submission.R +++ b/R/validate_submission.R @@ -47,6 +47,7 @@ run_all_validation <- function(df, path, js_def0, js_def, round_id, hub_path, all <- merge_sample_id(df, req_colnames, merge_sample_col, js_def0, js_def, checks, partition = partition, verbose = TRUE, verbose_col = verbose_col) + gc() df <- all$df pair <- all$msg if (!is.null(pair) && verbose) @@ -82,8 +83,10 @@ run_all_validation <- function(df, path, js_def0, js_def, round_id, hub_path, "tasks"), output_type_id_datatype = "from_config") + gc() tbl_chr <- dplyr::mutate_all(df, as.character) + gc() checks$valid_vals <- try_check(check_tbl_values(tbl_chr, round_id = round_id, file_path = file_path, hub_path = hub_path), @@ -92,26 +95,31 @@ run_all_validation <- function(df, path, js_def0, js_def, round_id, hub_path, return(checks) } + gc() checks$rows_unique <- try_check(check_tbl_rows_unique(tbl_chr, file_path = file_path, hub_path = hub_path), path) + gc() checks$req_vals <- try_check(check_df_values_required(df, js_def, file_path = file_path), path) # -- slow + gc() checks$value_col_valid <- try_check(check_tbl_value_col(df, round_id = round_id, file_path = file_path, hub_path = hub_path), path) - + gc() checks$value_col_non_desc <- try_check(check_tbl_value_col_ascending(tbl_chr, file_path = file_path, hub_path = hub_path, round_id = round_id, derived_task_ids = NULL), path) # -- slow + gc() checks <- sample_test(checks, tbl_chr, round_id, file_path, hub_path, path, pair, js_def) + gc() checks <- value_test(df, checks, file_path, n_decimal = n_decimal, pop = pop, obs = obs)