From 2df1442d327d951d42e0443ccdae3b6726618c93 Mon Sep 17 00:00:00 2001 From: wkliao Date: Wed, 31 Dec 2025 18:43:45 -0600 Subject: [PATCH 1/9] New hint nc_data_move_chunk_size When adding new data objects into an existing file, the data section may need to be moved to a higher file offset. The movement is performed in chunks. This hint allows users to customized the chunk size. The default is 1048576 bytes, i.e. 1 MiB. --- sneak_peek.md | 6 +- src/drivers/ncmpio/ncmpio_NC.h | 35 +++++-- src/drivers/ncmpio/ncmpio_enddef.c | 123 ++++++++++++------------- src/drivers/ncmpio/ncmpio_file_misc.c | 5 +- src/drivers/ncmpio/ncmpio_header_get.c | 40 ++++---- src/drivers/ncmpio/ncmpio_util.c | 38 ++++++-- 6 files changed, 148 insertions(+), 99 deletions(-) diff --git a/sneak_peek.md b/sneak_peek.md index 83360643a..4f1d97ac1 100644 --- a/sneak_peek.md +++ b/sneak_peek.md @@ -45,7 +45,11 @@ This is essentially a placeholder for the next release note ... + none * New PnetCDF hints - + none + + 'nc_data_move_chunk_size' -- When adding new data objects into an existing + file, the data section may need to be moved to a higher file offset. The + movement is performed in chunks. This hint allows users to customized the + chunk size. The default is 1048576 bytes, i.e. 1 MiB. + See [PR #203](https://github.com/Parallel-NetCDF/PnetCDF/pull/203). * New run-time environment variables + none diff --git a/src/drivers/ncmpio/ncmpio_NC.h b/src/drivers/ncmpio/ncmpio_NC.h index 4730ff373..6f011490a 100644 --- a/src/drivers/ncmpio/ncmpio_NC.h +++ b/src/drivers/ncmpio/ncmpio_NC.h @@ -18,13 +18,21 @@ #include "ncmpio_driver.h" #include "pncio.h" +/* default free space in the file header section. */ #define NC_DEFAULT_H_MINFREE 0 -#define NC_DEFAULT_V_ALIGN 512 + +/* default free space in the fix-sized variable section. */ #define NC_DEFAULT_V_MINFREE 0 + +/* default alignment for the starting offset of record variable section. */ #define NC_DEFAULT_R_ALIGN 4 -#define FILE_ALIGNMENT_DEFAULT 512 -#define FILE_ALIGNMENT_LB 4 +/* The default file header extent size is aligned with FILE_ALIGNMENT_DEFAULT. + * This default will be overwritten by hint nc_header_align_size or + * nc_var_align_size. Note when both hints are set by users, hint + * nc_var_align_size supersedes nc_header_align_size. + */ +#define NC_DEFAULT_V_ALIGN 512 /* MPI_OFFSET datatype was introduced in MPI 2.2 */ #if MPI_VERSION < 3 @@ -51,13 +59,25 @@ /* ncmpi_create/ncmpi_open set up header to be 'chunksize' big and to grow * by 'chunksize' as new items added. This used to be 4k. 256k lets us read - * in an entire climate header in one go */ -#define PNC_DEFAULT_CHUNKSIZE 262144 + * in an entire climate header in one go. This default will be overwritten by + * hint nc_header_read_chunk_size. + */ +#define PNC_HDR_READ_CHUNK_SIZE 262144 + +/* When file header grows or variables need to be moved to higher file offsets, + * data movement is performed in chunks of size PNC_MOVE_CHUNK_SIZE each per + * process. If the number of chunks is larger than the number of processes, + * carry out the data movement in multiple rounds. This default will be + * overwritten by hint nc_data_move_chunk_size. + */ +#define PNC_DATA_MOVE_CHUNK_SIZE 1048576 /* default size of temporal buffer to pack noncontiguous user buffers for MPI * collective read and write during ncmpi_wait/wait_all(). On some systems, * e.g. Cray KNL, using contiguous user buffers in collective I/O is much - * faster than noncontiguous. */ + * faster than noncontiguous. This default will be overwritten by hint + * nc_ibuf_size. + */ #define PNC_DEFAULT_IBUF_SIZE 16777216 /* when variable's nctype is NC_CHAR, I/O buffer's MPI type must be MPI_CHAR @@ -387,7 +407,8 @@ struct NC { struct NC *ncp_sf; /* ncp of subfile */ MPI_Comm comm_sf; /* subfile MPI communicator */ #endif - int chunk; /* chunk size for reading header, one chunk at a time */ + int hdr_chunk; /* chunk size for reading header, one chunk at a time */ + int data_chunk; /* chunk size for moving variables to higher offsets */ MPI_Offset v_align; /* alignment of the beginning of fixed-size variables */ MPI_Offset r_align; /* file alignment for record variable section */ MPI_Offset info_v_align;/* v_align set in MPI Info object */ diff --git a/src/drivers/ncmpio/ncmpio_enddef.c b/src/drivers/ncmpio/ncmpio_enddef.c index 6ffaa2eca..63de09ff9 100644 --- a/src/drivers/ncmpio/ncmpio_enddef.c +++ b/src/drivers/ncmpio/ncmpio_enddef.c @@ -30,12 +30,6 @@ #include "ncmpio_subfile.h" #endif -/* Divide the amount of data to be moved into chunks of size MOVE_UNIT each, - * and assign chunks to all processes. If the number of chunks is larger than - * the number of processes, carry out the data movement in multiple rounds. - */ -#define MOVE_UNIT 16777216 - #ifdef USE_POSIX_IO_TO_MOVE /*----< move_file_block() >-------------------------------------------------*/ /* Call POSIX I/O subroutines to move data */ @@ -65,13 +59,13 @@ move_file_block(NC *ncp, /* buf will be used as a temporal buffer to move data in chunks, i.e. * read a chunk and later write to the new location */ - buf = NCI_Malloc(MOVE_UNIT); + buf = NCI_Malloc(ncp->data_chunk); if (buf == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - p_units = (size_t)MOVE_UNIT * nprocs; + p_units = (size_t)ncp->data_chunk * nprocs; num_moves = nbytes / p_units; if (nbytes % p_units) num_moves++; - off_last = (num_moves - 1) * p_units + (size_t)rank * MOVE_UNIT; + off_last = (num_moves - 1) * p_units + (size_t)rank * ncp->data_chunk; off_from = from + off_last; off_to = to + off_last; mv_amnt = nbytes % p_units; @@ -82,8 +76,8 @@ move_file_block(NC *ncp, if (nbytes >= p_units) do_open = 1; else { - MPI_Offset n_units = nbytes / MOVE_UNIT; - if (nbytes % MOVE_UNIT) n_units++; + MPI_Offset n_units = nbytes / ncp->data_chunk; + if (nbytes % ncp->data_chunk) n_units++; if (rank < n_units) do_open = 1; } @@ -100,16 +94,16 @@ move_file_block(NC *ncp, if (mv_amnt == p_units) { /* each rank moves amount of chunk_size */ - chunk_size = MOVE_UNIT; + chunk_size = ncp->data_chunk; } else { /* when total move amount is less than p_units */ - size_t num_chunks = mv_amnt / MOVE_UNIT; - if (mv_amnt % MOVE_UNIT) num_chunks++; + size_t num_chunks = mv_amnt / ncp->data_chunk; + if (mv_amnt % ncp->data_chunk) num_chunks++; if (rank < num_chunks) { - chunk_size = MOVE_UNIT; - if (rank == num_chunks - 1 && mv_amnt % MOVE_UNIT > 0) - chunk_size = mv_amnt % MOVE_UNIT; + chunk_size = ncp->data_chunk; + if (rank == num_chunks - 1 && mv_amnt % ncp->data_chunk > 0) + chunk_size = mv_amnt % ncp->data_chunk; assert(chunk_size > 0); } else @@ -172,11 +166,10 @@ move_file_block(NC *ncp, { int rank, nprocs, status=NC_NOERR, do_coll; void *buf; - MPI_Offset num_moves, mv_amnt, p_units; + MPI_Offset last_block, mv_amnt, p_units; MPI_Offset off_last, off_from, off_to, rlen, wlen; MPI_Comm comm; - - if (to == from || nbytes == 0) return NC_NOERR; + PNCIO_View buf_view; /* If intra-node aggregation is enabled, then only the aggregators perform * the movement. @@ -188,10 +181,6 @@ move_file_block(NC *ncp, rank = (ncp->ina_comm == MPI_COMM_NULL) ? ncp->rank : ncp->ina_rank; nprocs = (ncp->ina_comm == MPI_COMM_NULL) ? ncp->nprocs : ncp->ina_nprocs; - /* MPI-IO fileview has been reset in ncmpi_redef() to make the entire file - * visible - */ - /* Use MPI collective I/O subroutines to move data, only if nproc > 1 and * MPI-IO hint "romio_no_indep_rw" is set to true. Otherwise, use MPI * independent I/O subroutines, as the data partitioned among processes are @@ -199,54 +188,53 @@ move_file_block(NC *ncp, */ do_coll = (nprocs > 1 && fIsSet(ncp->flags, NC_HCOLL)); + if (!do_coll && (to == from || nbytes == 0)) return NC_NOERR; + + /* MPI-IO fileview has been reset in ncmpi_redef() to make the entire file + * visible + */ + /* buf will be used as a temporal buffer to move data in chunks, i.e. * read a chunk and later write to the new location */ - buf = NCI_Malloc(MOVE_UNIT); + buf = NCI_Malloc(ncp->data_chunk); if (buf == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - p_units = (MPI_Offset)MOVE_UNIT * nprocs; - num_moves = nbytes / p_units; - if (nbytes % p_units) num_moves++; - off_last = (num_moves - 1) * p_units + (MPI_Offset)rank * MOVE_UNIT; + /* buffer used to move data is always contiguous */ + buf_view.type = MPI_BYTE; + buf_view.count = 0; + buf_view.is_contig = 1; + + /* movement must start from the last p_units toward to the 1st */ + p_units = (MPI_Offset)ncp->data_chunk * nprocs; + off_last = nbytes / p_units; + last_block = nbytes % p_units; + if (last_block == 0) { + off_last--; + last_block = p_units; + } + /* mv_amnt is the amount moved by this rank at the first round */ + if (rank < last_block / ncp->data_chunk) + mv_amnt = ncp->data_chunk; + else if (last_block % ncp->data_chunk > 0 && + rank == last_block / ncp->data_chunk) + mv_amnt = last_block % ncp->data_chunk; + else + mv_amnt = 0; + + off_last *= p_units; + off_last += (MPI_Offset)rank * ncp->data_chunk; off_from = from + off_last; off_to = to + off_last; - mv_amnt = nbytes % p_units; - if (mv_amnt == 0 && nbytes > 0) mv_amnt = p_units; - /* move the data section starting from its tail toward its beginning */ while (nbytes > 0) { - int chunk_size; + buf_view.size = mv_amnt; - if (mv_amnt == p_units) { - /* each rank moves amount of chunk_size */ - chunk_size = MOVE_UNIT; - } - else { - /* when total move amount is less than p_units */ - size_t num_chunks = mv_amnt / MOVE_UNIT; - if (mv_amnt % MOVE_UNIT) num_chunks++; - if (rank < num_chunks) { - chunk_size = MOVE_UNIT; - if (rank == num_chunks - 1 && mv_amnt % MOVE_UNIT > 0) - chunk_size = mv_amnt % MOVE_UNIT; - assert(chunk_size > 0); - } - else - chunk_size = 0; - } - - PNCIO_View buf_view; - buf_view.type = MPI_BYTE; - buf_view.size = chunk_size; - buf_view.count = 0; - buf_view.is_contig = 1; - - /* read from file at off_from for amount of chunk_size */ + /* read from file at off_from for amount of mv_amnt */ rlen = 0; if (do_coll) rlen = ncmpio_file_read_at_all(ncp, off_from, buf, buf_view); - else if (chunk_size > 0) + else if (mv_amnt > 0) rlen = ncmpio_file_read_at(ncp, off_from, buf, buf_view); if (status == NC_NOERR && rlen < 0) status = (int)rlen; @@ -265,10 +253,12 @@ move_file_block(NC *ncp, if (status == NC_NOERR && wlen < 0) status = (int)wlen; /* move on to the next round */ - mv_amnt = p_units; - off_from -= mv_amnt; - off_to -= mv_amnt; - nbytes -= mv_amnt; + nbytes -= p_units; + off_from -= p_units; + off_to -= p_units; + + /* mv_amnt becomes ncp->data_chunk in the 2nd and later rounds */ + mv_amnt = ncp->data_chunk; } NCI_Free(buf); @@ -426,7 +416,7 @@ NC_begins(NC *ncp, * creating a new file and first time call to ncmpi_enddef(), or other * case, e.g. opening an existing file, calling ncmpi_redef(), and then * ncmpi_enddef(). For the former case, ncp->begin_var == 0. For the latter - * case, ncp->begin_var must be > 0, as it is the orignial header extent. + * case, ncp->begin_var must be > 0, as it is the original header extent. * We increase begin_var only if the new header size grows out of its * original extent, or the start of variable section is not aligned as * requested by ncp->v_align. Note ncp->xsz is header size and @@ -612,6 +602,8 @@ write_NC(NC *ncp) assert(!NC_readonly(ncp)); buf_view.is_contig = 1; + buf_view.off = NULL; + buf_view.len = NULL; /* Depending on whether NC_HCOLL is set, writing file header can be done * through either MPI collective or independent write call. @@ -687,7 +679,7 @@ write_NC(NC *ncp) remain = header_wlen; buf_ptr = buf; buf_view.type = MPI_BYTE; - buf_view.count = 1; + buf_view.count = 0; for (i=0; ir_align); MPI_Info_set(*info_used, "nc_record_align_size", value); - sprintf(value, "%d", ncp->chunk); + sprintf(value, "%d", ncp->hdr_chunk); MPI_Info_set(*info_used, "nc_header_read_chunk_size", value); + sprintf(value, "%d", ncp->data_chunk); + MPI_Info_set(*info_used, "nc_data_move_chunk_size", value); + if (fIsSet(ncp->flags, NC_MODE_SWAP_ON)) MPI_Info_set(*info_used, "nc_in_place_swap", "enable"); else if (fIsSet(ncp->flags, NC_MODE_SWAP_OFF)) diff --git a/src/drivers/ncmpio/ncmpio_header_get.c b/src/drivers/ncmpio/ncmpio_header_get.c index bd91dd5d7..bfda5592b 100644 --- a/src/drivers/ncmpio/ncmpio_header_get.c +++ b/src/drivers/ncmpio/ncmpio_header_get.c @@ -323,8 +323,8 @@ hdr_len_NC_vararray(const NC_vararray *ncap, /*----< hdr_fetch() >--------------------------------------------------------*/ /* Fetch the next header chunk. The chunk buffer, pointed by gbp->base, is of - * size 'gbp->ncp->chunk' bytes. Be careful not to overwrite leftover (yet to - * be used) data in the buffer before fetching a new chunk. + * size 'gbp->ncp->hdr_chunk' bytes. Be careful not to overwrite leftover (yet + * to be used) data in the buffer before fetching a new chunk. */ static int hdr_fetch(bufferinfo *gbp) { @@ -348,17 +348,17 @@ hdr_fetch(bufferinfo *gbp) { MPI_Offset rlen; /* any leftover data in the buffer */ - slack = gbp->ncp->chunk - (gbp->pos - gbp->base); - if (slack == gbp->ncp->chunk) slack = 0; + slack = gbp->ncp->hdr_chunk - (gbp->pos - gbp->base); + if (slack == gbp->ncp->hdr_chunk) slack = 0; - /* When gbp->ncp->chunk == (gbp->pos - gbp->base), all data in the + /* When gbp->ncp->hdr_chunk == (gbp->pos - gbp->base), all data in the * buffer has been consumed. If not, then read additional header of - * size (gbp->ncp->chunk - slack) into a contiguous buffer, pointed by - * gbp->base + slack. + * size (gbp->ncp->hdr_chunk - slack) into a contiguous buffer, pointed + * by gbp->base + slack. */ readBuf = gbp->base; - readLen = gbp->ncp->chunk; + readLen = gbp->ncp->hdr_chunk; if (slack > 0) { /* move slack to beginning of the buffer, gbp->base */ memmove(gbp->base, gbp->pos, slack); readBuf += slack; @@ -383,8 +383,8 @@ hdr_fetch(bufferinfo *gbp) { * when the remaining file size is smaller than readLen. When * actual read amount is smaller than readLen, then we zero-out the * remaining buffer. This is because the MPI_Bcast below broadcasts - * a buffer of a fixed size, gbp->ncp->chunk. Without zeroing out, - * valgrind will complain about the uninitialized values. + * a buffer of a fixed size, gbp->ncp->hdr_chunk. Without zeroing + * out, valgrind will complain about the uninitialized values. */ if (rlen < readLen) memset(readBuf + rlen, 0, readLen - rlen); @@ -414,7 +414,7 @@ hdr_fetch(bufferinfo *gbp) { /* broadcast root's read (full or partial header) to other processes */ if (nprocs > 1) { - TRACE_COMM(MPI_Bcast)(gbp->base, gbp->ncp->chunk, MPI_BYTE, 0, + TRACE_COMM(MPI_Bcast)(gbp->base, gbp->ncp->hdr_chunk, MPI_BYTE, 0, gbp->ncp->comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast"); @@ -570,7 +570,7 @@ hdr_get_NC_name(bufferinfo *gbp, char **namep, size_t *name_len) */ padding = PNETCDF_RNDUP(nchars, X_ALIGN) - nchars; - bufremain = gbp->ncp->chunk - (gbp->pos - gbp->base); + bufremain = gbp->ncp->hdr_chunk - (gbp->pos - gbp->base); cpos = *namep; @@ -591,7 +591,7 @@ hdr_get_NC_name(bufferinfo *gbp, char **namep, size_t *name_len) *namep = NULL; return err; } - bufremain = gbp->ncp->chunk; + bufremain = gbp->ncp->hdr_chunk; } } @@ -815,8 +815,8 @@ hdr_get_NC_attrV(bufferinfo *gbp, NC_attr *attrp) nbytes = attrp->nelems * xsz; padding = attrp->xsz - nbytes; - bufremain = gbp->ncp->chunk - (gbp->pos - gbp->base); - /* gbp->ncp->chunk is the read chunk size, which is of type 4-byte int. + bufremain = gbp->ncp->hdr_chunk - (gbp->pos - gbp->base); + /* gbp->ncp->hdr_chunk is the read chunk size, which is of type 4-byte int. * thus bufremain should be less than INT_MAX */ /* get values */ @@ -831,7 +831,7 @@ hdr_get_NC_attrV(bufferinfo *gbp, NC_attr *attrp) } else { err = hdr_fetch(gbp); if (err != NC_NOERR) return err; - bufremain = gbp->ncp->chunk; + bufremain = gbp->ncp->hdr_chunk; } } @@ -1346,11 +1346,13 @@ ncmpio_hdr_get_NC(NC *ncp) /* Initialize the get buffer that stores the header read from the file */ getbuf.ncp = ncp; getbuf.offset = 0; /* read from start of the file */ - getbuf.base = (char*) NCI_Malloc(getbuf.ncp->chunk); + getbuf.base = (char*) NCI_Malloc(getbuf.ncp->hdr_chunk); getbuf.pos = getbuf.base; - getbuf.end = getbuf.base + getbuf.ncp->chunk; + getbuf.end = getbuf.base + getbuf.ncp->hdr_chunk; - /* Fetch the next header chunk. The chunk is 'gbp->ncp->chunk' bytes big */ + /* Fetch the next header chunk. The chunk is 'gbp->ncp->hdr_chunk' bytes + * big. + */ err = hdr_fetch(&getbuf); if (err != NC_NOERR) return err; diff --git a/src/drivers/ncmpio/ncmpio_util.c b/src/drivers/ncmpio/ncmpio_util.c index 3e54c9b81..ad9fcde61 100644 --- a/src/drivers/ncmpio/ncmpio_util.c +++ b/src/drivers/ncmpio/ncmpio_util.c @@ -42,7 +42,10 @@ void ncmpio_hint_extract(NC *ncp, ncp->info_r_align = -1; /* -1 indicates not set */ /* chunk size for reading header (set default before check hints) */ - ncp->chunk = PNC_DEFAULT_CHUNKSIZE; + ncp->hdr_chunk = PNC_HDR_READ_CHUNK_SIZE; + + /* chunk size for moving variables to higher offsets */ + ncp->data_chunk = PNC_DATA_MOVE_CHUNK_SIZE; /* buffer to pack noncontiguous user buffers when calling wait() */ ncp->ibuf_size = PNC_DEFAULT_IBUF_SIZE; @@ -121,14 +124,15 @@ void ncmpio_hint_extract(NC *ncp, llval = strtoll(value, NULL, 10); if (errno == 0) { if (llval < 0) - ncp->chunk = 0; + ncp->hdr_chunk = PNC_HDR_READ_CHUNK_SIZE; else if (llval > NC_MAX_INT) /* limit to NC_MAX_INT */ - ncp->chunk = NC_MAX_INT; + ncp->hdr_chunk = NC_MAX_INT; else - ncp->chunk = (int)llval; + ncp->hdr_chunk = (int)llval; /* CDF-5's minimum header size is 4 bytes more than CDF-1 2's */ - ncp->chunk = PNETCDF_RNDUP(MAX(MIN_NC_XSZ+4, ncp->chunk), X_ALIGN); + ncp->hdr_chunk = PNETCDF_RNDUP(MAX(MIN_NC_XSZ+4, ncp->hdr_chunk), + X_ALIGN); } } @@ -241,6 +245,24 @@ void ncmpio_hint_extract(NC *ncp, MPI_Info_get(info, "romio_no_indep_rw", MPI_MAX_INFO_VAL-1, value, &flag); if (flag && strcasecmp(value, "true") == 0) fSet(ncp->flags, NC_HCOLL); + + /* Data movement chunk size when variables need to be moved to higher file + * offsets. + */ + MPI_Info_get(info, "nc_data_move_chunk_size", MPI_MAX_INFO_VAL-1, value, + &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + llval = strtoll(value, NULL, 10); + if (errno == 0) { + if (llval < 0) + ncp->data_chunk = PNC_DATA_MOVE_CHUNK_SIZE; + else if (llval > NC_MAX_INT) /* limit to NC_MAX_INT */ + ncp->data_chunk = NC_MAX_INT; + else + ncp->data_chunk = (int)llval; + } + } } /*----< ncmpio_hint_set() >--------------------------------------------------*/ @@ -272,9 +294,13 @@ void ncmpio_hint_set(NC *ncp, } /* header reading chunk size */ - snprintf(int_str, MAX_INT_LEN, "%d", ncp->chunk); + snprintf(int_str, MAX_INT_LEN, "%d", ncp->hdr_chunk); MPI_Info_set(info, "nc_header_read_chunk_size", int_str); + /* variable movement chunk size */ + snprintf(int_str, MAX_INT_LEN, "%d", ncp->data_chunk); + MPI_Info_set(info, "nc_data_move_chunk_size", int_str); + /* setting in-place byte swap (matters only for Little Endian) */ int swap_on = fIsSet(ncp->flags, NC_MODE_SWAP_ON); int swap_off = fIsSet(ncp->flags, NC_MODE_SWAP_OFF); From b8dc3a69643bbedb009b8fb706b9cc7b1f375ab2 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Thu, 1 Jan 2026 13:07:47 -0800 Subject: [PATCH 2/9] wrf_io.c print hint nc_data_move_chunk_size --- benchmarks/WRF-IO/wrf_io.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/benchmarks/WRF-IO/wrf_io.c b/benchmarks/WRF-IO/wrf_io.c index ea3ce2df8..85726e16c 100644 --- a/benchmarks/WRF-IO/wrf_io.c +++ b/benchmarks/WRF-IO/wrf_io.c @@ -1190,13 +1190,14 @@ int wrf_r_benchmark(char *in_file, static int grow_header_benchmark(char *in_file) { - char *attr; + char value[MPI_MAX_INFO_VAL], *attr; int i, err=NC_NOERR, nprocs, rank, ncid, ndims, dimid[3]; - int varid, unlimdimid, nvars, fix_nvars, rec_nvars; + int varid, unlimdimid, nvars, fix_nvars, rec_nvars, len, flag; double timing, max_t; MPI_Offset hdr_size, hdr_extent, attr_len, num_rec, longitude, latitude; MPI_Offset r_amnt[2], w_amnt[2], amnt[2], sum_amnt[2], fix_off, rec_off; - MPI_Offset rec_size; + MPI_Offset rec_size, nc_data_move_chunk_size; + MPI_Info info; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); @@ -1339,6 +1340,18 @@ int grow_header_benchmark(char *in_file) CHECK_ERR("ncmpi_fill_var_rec") } + err = ncmpi_inq_file_info(ncid, &info); + CHECK_ERR("ncmpi_inq_file_info") + + MPI_Info_get_valuelen(info, "nc_data_move_chunk_size", &len, &flag); + if (flag) { + MPI_Info_get(info, "nc_data_move_chunk_size", len+1, value, &flag); + nc_data_move_chunk_size = strtoll(value, NULL, 10); + } else + nc_data_move_chunk_size = 0; + + MPI_Info_free(&info); + /* close file */ err = ncmpi_close(ncid); CHECK_ERR("ncmpi_close") @@ -1370,6 +1383,7 @@ int grow_header_benchmark(char *in_file) printf("Max time: %.4f sec\n", max_t); printf("Write bandwidth: %.2f MiB/s\n", bw/max_t); printf(" %.2f GiB/s\n", bw/1024.0/max_t); + printf("Hint nc_data_move_chunk_size %lld\n", nc_data_move_chunk_size); printf("-----------------------------------------------------------\n"); } From 784adab9b8ea90badbb055d5c2656d0cbcfe740e Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Thu, 1 Jan 2026 13:09:21 -0800 Subject: [PATCH 3/9] test/parallel_run.sh - set nc_data_move_chunk_size to 100 In addition, define DRY_RUN and VERBOSE and 2 functions for debugging purpose. --- test/parallel_run.sh | 148 +++++++++++++++++++++---------------------- 1 file changed, 71 insertions(+), 77 deletions(-) diff --git a/test/parallel_run.sh b/test/parallel_run.sh index 33e307987..f5e180a6b 100755 --- a/test/parallel_run.sh +++ b/test/parallel_run.sh @@ -7,6 +7,29 @@ # Exit immediately if a command exits with a non-zero status. set -e +DRY_RUN=no +VERBOSE=no + +exe_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} + +seq_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} + VALIDATOR=../../src/utils/ncvalidator/ncvalidator NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff @@ -33,26 +56,27 @@ unset PNETCDF_HINTS fixed_length=23 for i in ${check_PROGRAMS} ; do + if test "$i" = pres_temp_4D_rd ; then + # running pres_temp_4D_rd is a part of pres_temp_4D_wr + continue + fi if test "$i" = tst_io ; then # this is designed to run 1 process continue fi if test "$i" = tst_version ; then # this program read only and creates no output file - # echo "${LINENO}: ${MPIRUN} ./$i" - ${MPIRUN} ./$i + exe_cmd ./$i continue fi if test "$i" = tst_open_cdf5 ; then # this program read only and creates no output file - # echo "${LINENO}: ${MPIRUN} ./$i ${srcdir}/bad_begin.nc5" - ${MPIRUN} ./$i ${srcdir}/bad_begin.nc5 + exe_cmd ./$i ${srcdir}/bad_begin.nc5 continue fi if test "$i" = tst_corrupt ; then # this program read only and creates no output file - # echo "${LINENO}: ${MPIRUN} ./$i ${srcdir}" - ${MPIRUN} ./$i ${srcdir} + exe_cmd ./$i ${srcdir} continue fi @@ -98,7 +122,11 @@ for i in ${check_PROGRAMS} ; do fi fi - PNETCDF_HINTS=$no_indep_rw_hint + # More rigorous tests using a small moving chunk size + PNETCDF_HINTS="nc_data_move_chunk_size=100" + + PNETCDF_HINTS="$no_indep_rw_hint;$PNETCDF_HINTS" + if test "x$SAFE_HINTS" != x ; then PNETCDF_HINTS="$SAFE_HINTS;$PNETCDF_HINTS" fi @@ -114,59 +142,47 @@ for i in ${check_PROGRAMS} ; do export PNETCDF_HINTS="$PNETCDF_HINTS" export PNETCDF_SAFE_MODE=$j - # echo "${LINENO}: PNETCDF_SAFE_MODE=$PNETCDF_SAFE_MODE PNETCDF_HINTS=$PNETCDF_HINTS" + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line ${LINENO}: PNETCDF_SAFE_MODE=$PNETCDF_SAFE_MODE PNETCDF_HINTS=$PNETCDF_HINTS" + fi if test "$i" = tst_pthread ; then # each MPI process created 6 threads - # echo "${LINENO}: ${MPIRUN} ./$i ${OUT_FILE}.nc" - ${MPIRUN} ./$i ${OUT_FILE}.nc + exe_cmd ./$i ${OUT_FILE}.nc for k in `seq 0 ${NTHREADS}` ; do - # echo "${LINENO}: ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.nc.$k" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.nc.$k + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc.$k rm -f ${OUTDIR}/tst_pthread.nc.$k done continue elif test "$i" = pres_temp_4D_wr ; then - # echo "${LINENO}: ${MPIRUN} ./$i ${OUT_FILE}.nc" - ${MPIRUN} ./$i ${OUT_FILE}.nc - # echo "${LINENO}: ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.nc - - # echo "${LINENO}: ${MPIRUN} ./pres_temp_4D_rd ${OUT_FILE}.nc" - ${MPIRUN} ./pres_temp_4D_rd ${OUT_FILE}.nc + exe_cmd ./$i ${OUT_FILE}.nc + exe_cmd ./pres_temp_4D_rd ${OUT_FILE}.nc elif test "$i" = pres_temp_4D_rd ; then continue elif test "$i" = test_inq_format ; then - # echo "${LINENO}: ${MPIRUN} ./$i ${srcdir}" - ${MPIRUN} ./$i ${srcdir} + exe_cmd ./$i ${srcdir} continue elif test "$i" = "tst_cdl_hdr_parser" ; then - # echo "${LINENO}: ${MPIRUN} ./$i -q -o ${OUT_FILE}.nc ${srcdir}/cdl_header.txt" - ${MPIRUN} ./$i -q -o ${OUT_FILE}.nc ${srcdir}/cdl_header.txt + exe_cmd ./$i -q -o ${OUT_FILE}.nc ${srcdir}/cdl_header.txt continue elif test "$i" = mcoll_perf ; then - # echo "${LINENO}: ${MPIRUN} ./$i ${OUT_FILE}" - ${MPIRUN} ./$i ${OUT_FILE} + exe_cmd ./$i ${OUT_FILE} else - # echo "${LINENO}: ${MPIRUN} ./$i ${OUT_FILE}.nc" - ${MPIRUN} ./$i ${OUT_FILE}.nc + exe_cmd ./$i ${OUT_FILE}.nc fi # put_all_kinds and iput_all_kinds output 3 files if test "$i" = put_all_kinds -o "$i" = iput_all_kinds ; then for k in 1 2 5 ; do - # echo "${LINENO}: --- validating file ${OUT_FILE}.nc$k" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.nc$k + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc$k done elif test "$i" = mcoll_perf ; then for j in `seq 0 9` ; do ext="2.4.$j.nc" - # echo "${LINENO}:--- validating file ${OUT_FILE}.$ext" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.$ext + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.$ext done else - # echo "${LINENO}:--- validating file ${OUT_FILE}.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.nc + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc fi if test "x${ENABLE_BURST_BUFFER}" = x1 ; then @@ -174,36 +190,29 @@ for i in ${check_PROGRAMS} ; do saved_PNETCDF_HINTS=${PNETCDF_HINTS} export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" if test "$i" = mcoll_perf ; then - # echo "${LINENO}:--- ${MPIRUN} ./$i ${OUT_FILE}.bb" - ${MPIRUN} ./$i ${OUT_FILE}.bb + exe_cmd ./$i ${OUT_FILE}.bb else - # echo "${LINENO}:--- ${MPIRUN} ./$i ${OUT_FILE}.bb.nc" - ${MPIRUN} ./$i ${OUT_FILE}.bb.nc + exe_cmd ./$i ${OUT_FILE}.bb.nc fi export PNETCDF_HINTS=${saved_PNETCDF_HINTS} # put_all_kinds and iput_all_kinds output 3 files if test "$i" = put_all_kinds -o "$i" = iput_all_kinds ; then for k in 1 2 5 ; do - # echo "${LINENO}: --- validating file ${OUT_FILE}.bb.nc$k" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.bb.nc$k - # echo "${LINENO}: --- ncmpidiff ${OUT_FILE}.nc$k ${OUT_FILE}.bb.nc$k ---" - ${MPIRUN} ${NCMPIDIFF} -q ${OUT_FILE}.nc$k ${OUT_FILE}.bb.nc$k + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.bb.nc$k + exe_cmd ${NCMPIDIFF} -q ${OUT_FILE}.nc$k ${OUT_FILE}.bb.nc$k done continue elif test "$i" = mcoll_perf ; then for j in `seq 0 9` ; do ext="2.4.$j.nc" bb_ext="bb.2.4.$j.nc" - # echo "${LINENO}:--- validating file ${OUT_FILE}.$bb_ext" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.$bb_ext - # echo "${LINENO}:--- ncmpidiff -q ${OUT_FILE}.$ext ${OUT_FILE}.$bb_ext" - ${MPIRUN} ${NCMPIDIFF} -q ${OUT_FILE}.$ext ${OUT_FILE}.$bb_ext + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.$bb_ext + exe_cmd ${NCMPIDIFF} -q ${OUT_FILE}.$ext ${OUT_FILE}.$bb_ext done continue else - # echo "${LINENO}: --- validating file ${OUT_FILE}.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.bb.nc + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.bb.nc fi # compare file header only for large file tests @@ -227,17 +236,14 @@ for i in ${check_PROGRAMS} ; do test "$i" = tst_hash_large_nvars ; then DIFF_OPT+=" -h" fi - # echo "${LINENO}: --- ncmpidiff $DIFF_OPT $OUT_FILE.nc $OUT_FILE.bb.nc ---" - ${MPIRUN} ${NCMPIDIFF} $DIFF_OPT $OUT_FILE.nc $OUT_FILE.bb.nc + exe_cmd ${NCMPIDIFF} $DIFF_OPT $OUT_FILE.nc $OUT_FILE.bb.nc fi if test "x${ENABLE_NETCDF4}" = x1 ; then if test "$i" = tst_grow_data ; then continue fi - # echo "${LINENO}: test netCDF-4 feature" - # echo "${LINENO}: ${MPIRUN} ./$i ${OUT_FILE}.nc4 4" - ${MPIRUN} ./$i ${OUT_FILE}.nc4 4 + exe_cmd ./$i ${OUT_FILE}.nc4 4 # Validator does not support nc4 fi done # intra_aggr @@ -267,44 +273,32 @@ for i in ${check_PROGRAMS} ; do test "$i" = tst_hash_large_nvars ; then DIFF_OPT+=" -h" fi - if test "$i" = pres_temp_4D_rd || test "$i" = test_inq_format ; then + if test "$i" = test_inq_format ; then continue fi if test "$i" = put_all_kinds || test "$i" = iput_all_kinds ; then for j in 1 2 5; do - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.mpio.ina.nc$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.mpio.ina.nc$j - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.pncio.nc$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.pncio.nc$j - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.nc$j $OUT_PREFIX.pncio.ina.nc$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc$j $OUT_PREFIX.pncio.ina.nc$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.mpio.ina.nc$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.pncio.nc$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc$j $OUT_PREFIX.pncio.ina.nc$j done elif test "$i" = tst_pthread ; then for j in `seq 0 ${NTHREADS}` ; do - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.mpio.ina.nc.$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.mpio.ina.nc.$j - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.nc.$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.nc.$j - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.mpio.ina.nc.$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.nc.$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j done elif test "$i" = mcoll_perf ; then for j in `seq 0 9` ; do ext="2.4.$j.nc" - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.$ext $OUT_PREFIX.mpio.ina.$ext ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.$ext $OUT_PREFIX.mpio.ina.$ext - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.$ext $OUT_PREFIX.pncio.$ext ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.$ext $OUT_PREFIX.pncio.$ext - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.$ext $OUT_PREFIX.pncio.ina.$ext ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.$ext $OUT_PREFIX.pncio.ina.$ext + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.$ext $OUT_PREFIX.mpio.ina.$ext + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.$ext $OUT_PREFIX.pncio.$ext + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.$ext $OUT_PREFIX.pncio.ina.$ext done else - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc fi done # no_indep_rw From d9ba4cadc9d0a45309371f72fb948ec1e007a4cb Mon Sep 17 00:00:00 2001 From: wkliao Date: Thu, 1 Jan 2026 17:28:47 -0600 Subject: [PATCH 4/9] Github actions: separate tests of MIMIC_LUSTRE --- .github/workflows/mac_mpich.yml | 37 ---- .github/workflows/mac_mpich_lustre.yml | 170 ++++++++++++++++++ .github/workflows/mac_openmpi.yml | 37 ---- .github/workflows/mac_openmpi_lustre.yml | 172 ++++++++++++++++++ .github/workflows/ubuntu_mpich.yml | 36 +--- .github/workflows/ubuntu_mpich_lustre.yml | 184 ++++++++++++++++++++ .github/workflows/ubuntu_openmpi.yml | 36 ---- .github/workflows/ubuntu_openmpi_adios.yml | 37 ---- .github/workflows/ubuntu_openmpi_lustre.yml | 178 +++++++++++++++++++ 9 files changed, 706 insertions(+), 181 deletions(-) create mode 100644 .github/workflows/mac_mpich_lustre.yml create mode 100644 .github/workflows/mac_openmpi_lustre.yml create mode 100644 .github/workflows/ubuntu_mpich_lustre.yml create mode 100644 .github/workflows/ubuntu_openmpi_lustre.yml diff --git a/.github/workflows/mac_mpich.yml b/.github/workflows/mac_mpich.yml index 2010f9e24..c50885758 100644 --- a/.github/workflows/mac_mpich.yml +++ b/.github/workflows/mac_mpich.yml @@ -115,7 +115,6 @@ jobs: --with-pthread \ --disable-fortran \ --with-mpi=${GITHUB_WORKSPACE}/MPICH \ - MIMIC_LUSTRE=yes \ TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output make -j 8 tests - name: Print config.log @@ -142,42 +141,6 @@ jobs: run: | cd ${GITHUB_WORKSPACE} make ptests - - name: Build PnetCDF (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - make distclean - rm -rf pnetcdf_output - mkdir -p pnetcdf_output - ./configure --disable-fortran \ - --with-mpi=${GITHUB_WORKSPACE}/MPICH \ - TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output - make -j 8 tests - - name: Print config.log (default configuration) - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files (default configuration) - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make ptests - name: make distcheck run: | cd ${GITHUB_WORKSPACE} diff --git a/.github/workflows/mac_mpich_lustre.yml b/.github/workflows/mac_mpich_lustre.yml new file mode 100644 index 000000000..c2f7fe8b5 --- /dev/null +++ b/.github/workflows/mac_mpich_lustre.yml @@ -0,0 +1,170 @@ +name: Mac OSX with MPICH (MIMIC_LUSTRE) + +on: + push: + branches: [ master, test_github_actions ] + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - '**/*.jpg' + - '**/*.png' + - 'docs/*' + - 'test/test_installed/*' + pull_request: + branches: [ master, test_github_actions ] + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - '**/*.jpg' + - '**/*.png' + - 'docs/*' + - 'test/test_installed/*' + +env: + MPICH_VERSION: 4.3.0 + AUTOCONF_VERSION: 2.71 + AUTOMAKE_VERSION: 1.17 + LIBTOOL_VERSION: 2.5.4 + M4_VERSION: 1.4.19 + +jobs: + build: + runs-on: macos-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - name: Set up dependencies + run: | + # brew install gcc + # which gcc + # gcc --version + # which gfortran + - name: Clean up git untracked files + run: | + git clean -fx + - name: Build GNU autotools + run: | + export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" + export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz + gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf - + cd m4-${M4_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz + gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf - + cd autoconf-${AUTOCONF_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz + gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf - + cd automake-${AUTOMAKE_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz + gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - + cd libtool-${LIBTOOL_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + - name: Build MPICH + run: | + cd ${GITHUB_WORKSPACE} + rm -rf MPICH ; mkdir MPICH ; cd MPICH + wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz + gzip -dc mpich-${MPICH_VERSION}.tar.gz | tar -xf - + cd mpich-${MPICH_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/MPICH \ + --silent \ + --enable-romio \ + --with-file-system=ufs \ + --with-device=ch3:sock \ + --disable-fortran \ + CC=gcc + make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + - name: Build PnetCDF with MIMIC_LUSTRE + run: | + cd ${GITHUB_WORKSPACE} + export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" + export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" + m4 --version + autoconf --version + automake --version + libtool --version + autoreconf -i + mkdir -p pnetcdf_output + ./configure --enable-option-checking=fatal \ + pnc_ac_debug=yes \ + --enable-burst_buffering \ + --enable-subfiling \ + --enable-thread-safe \ + --with-pthread \ + --disable-fortran \ + --with-mpi=${GITHUB_WORKSPACE}/MPICH \ + TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output \ + MIMIC_LUSTRE=yes + make -j 8 tests + - name: Print config.log + if: ${{ always() }} + run: | + cat ${GITHUB_WORKSPACE}/config.log + - name: make check + run: | + cd ${GITHUB_WORKSPACE} + make check + - name: Print test log files + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + fname=`find src test examples benchmarks -type f -name "*.log"` + for f in $fname ; do \ + bname=`basename $f` ; \ + if test "x$bname" != xconfig.log ; then \ + echo "-------- dump $f ----------------------------" ; \ + cat $f ; \ + fi ; \ + done + - name: make ptests + run: | + cd ${GITHUB_WORKSPACE} + make ptests + - name: make distcheck + run: | + cd ${GITHUB_WORKSPACE} + make -j 8 distcheck DISTCHECK_CONFIGURE_FLAGS="--silent --with-mpi=${GITHUB_WORKSPACE}/MPICH" + - name: make install + run: | + cd ${GITHUB_WORKSPACE} + prefix_path=${GITHUB_WORKSPACE}/pnetcdf_install + echo "---- test make install prefix=${prefix_path}" + make install prefix=${prefix_path} + test/tst_install.sh ${prefix_path} + prefix_path="/pnetcdf_install" + destdir_path=${GITHUB_WORKSPACE}/inst + echo "---- test make install prefix=${prefix_path} DESTDIR=${destdir_path}" + make install prefix=${prefix_path} DESTDIR=${destdir_path} + test/tst_install.sh ${prefix_path} ${destdir_path} + - name: Cleanup + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + make -s distclean + rm -rf ${GITHUB_WORKSPACE}/pnetcdf_output + rm -rf ${GITHUB_WORKSPACE}/MPICH + rm -rf ${GITHUB_WORKSPACE}/pnetcdf_install + rm -rf ${GITHUB_WORKSPACE}/inst + diff --git a/.github/workflows/mac_openmpi.yml b/.github/workflows/mac_openmpi.yml index 8eea60307..158af438b 100644 --- a/.github/workflows/mac_openmpi.yml +++ b/.github/workflows/mac_openmpi.yml @@ -117,7 +117,6 @@ jobs: --with-pthread \ --disable-fortran \ --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ - MIMIC_LUSTRE=yes \ TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output make -j 8 tests - name: Print config.log @@ -144,42 +143,6 @@ jobs: run: | cd ${GITHUB_WORKSPACE} make ptests - - name: Build PnetCDF (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - make distclean - rm -rf pnetcdf_output - mkdir -p pnetcdf_output - ./configure --disable-fortran \ - --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ - TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output - make -j 8 tests - - name: Print config.log (default configuration) - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files (default configuration) - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make ptests - name: make distcheck run: | cd ${GITHUB_WORKSPACE} diff --git a/.github/workflows/mac_openmpi_lustre.yml b/.github/workflows/mac_openmpi_lustre.yml new file mode 100644 index 000000000..303a22a66 --- /dev/null +++ b/.github/workflows/mac_openmpi_lustre.yml @@ -0,0 +1,172 @@ +name: Mac OSX with OpenMPI (MIMIC_LUSTRE) + +on: + push: + branches: [ master, test_github_actions ] + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - '**/*.jpg' + - '**/*.png' + - 'docs/*' + - 'test/test_installed/*' + pull_request: + branches: [ master, test_github_actions ] + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - '**/*.jpg' + - '**/*.png' + - 'docs/*' + - 'test/test_installed/*' + +env: + OPENMPI_VERSION: 5.0.8 + AUTOCONF_VERSION: 2.71 + AUTOMAKE_VERSION: 1.17 + LIBTOOL_VERSION: 2.5.4 + M4_VERSION: 1.4.19 + +jobs: + build: + runs-on: macos-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - name: Set up dependencies + run: | + # brew install gcc + # which gcc + # gcc --version + # which gfortran + - name: Clean up git untracked files + run: | + git clean -fx + - name: Build GNU autotools + run: | + export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" + export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz + gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf - + cd m4-${M4_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz + gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf - + cd autoconf-${AUTOCONF_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz + gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf - + cd automake-${AUTOMAKE_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz + gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - + cd libtool-${LIBTOOL_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + - name: Build OPENMPI + run: | + cd ${GITHUB_WORKSPACE} + rm -rf OPENMPI ; mkdir OPENMPI ; cd OPENMPI + VER_MAJOR=${OPENMPI_VERSION%.*} + wget -q https://download.open-mpi.org/release/open-mpi/v${VER_MAJOR}/openmpi-${OPENMPI_VERSION}.tar.gz + gzip -dc openmpi-${OPENMPI_VERSION}.tar.gz | tar -xf - + cd openmpi-${OPENMPI_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/OPENMPI \ + --silent \ + --with-io-romio-flags="--with-file-system=ufs" \ + --with-hwloc=internal \ + --with-pmix=internal \ + --with-libevent=internal \ + --disable-mpi-fortran \ + CC=gcc + make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + - name: Build PnetCDF (MIMIC_LUSTRE) + run: | + cd ${GITHUB_WORKSPACE} + export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" + export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" + m4 --version + autoconf --version + automake --version + libtool --version + autoreconf -i + mkdir -p pnetcdf_output + ./configure --enable-option-checking=fatal \ + pnc_ac_debug=yes \ + --enable-burst_buffering \ + --enable-subfiling \ + --enable-thread-safe \ + --with-pthread \ + --disable-fortran \ + --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ + TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output \ + MIMIC_LUSTRE=yes + make -j 8 tests + - name: Print config.log + if: ${{ always() }} + run: | + cat ${GITHUB_WORKSPACE}/config.log + - name: make check + run: | + cd ${GITHUB_WORKSPACE} + make check + - name: Print test log files + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + fname=`find src test examples benchmarks -type f -name "*.log"` + for f in $fname ; do \ + bname=`basename $f` ; \ + if test "x$bname" != xconfig.log ; then \ + echo "-------- dump $f ----------------------------" ; \ + cat $f ; \ + fi ; \ + done + - name: make ptests + run: | + cd ${GITHUB_WORKSPACE} + make ptests + - name: make distcheck + run: | + cd ${GITHUB_WORKSPACE} + make -j 8 distcheck DISTCHECK_CONFIGURE_FLAGS="--silent --with-mpi=${GITHUB_WORKSPACE}/OPENMPI" + - name: make install + run: | + cd ${GITHUB_WORKSPACE} + prefix_path=${GITHUB_WORKSPACE}/pnetcdf_install + echo "---- test make install prefix=${prefix_path}" + make install prefix=${prefix_path} + test/tst_install.sh ${prefix_path} + prefix_path="/pnetcdf_install" + destdir_path=${GITHUB_WORKSPACE}/inst + echo "---- test make install prefix=${prefix_path} DESTDIR=${destdir_path}" + make install prefix=${prefix_path} DESTDIR=${destdir_path} + test/tst_install.sh ${prefix_path} ${destdir_path} + - name: Cleanup + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + make -s distclean + rm -rf ${GITHUB_WORKSPACE}/pnetcdf_output + rm -rf ${GITHUB_WORKSPACE}/OPENMPI + rm -rf ${GITHUB_WORKSPACE}/pnetcdf_install + rm -rf ${GITHUB_WORKSPACE}/inst + diff --git a/.github/workflows/ubuntu_mpich.yml b/.github/workflows/ubuntu_mpich.yml index f51bb9be1..6126e78ee 100644 --- a/.github/workflows/ubuntu_mpich.yml +++ b/.github/workflows/ubuntu_mpich.yml @@ -120,6 +120,7 @@ jobs: automake --version libtool --version autoreconf -i + mkdir -p pnetcdf_output ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ --enable-option-checking=fatal \ pnc_ac_debug=yes \ @@ -128,7 +129,7 @@ jobs: --enable-thread-safe \ --with-pthread \ --with-mpi=${GITHUB_WORKSPACE}/MPICH \ - MIMIC_LUSTRE=yes + TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output make -j 8 tests - name: Print config.log if: ${{ always() }} @@ -154,39 +155,6 @@ jobs: run: | cd ${GITHUB_WORKSPACE} make ptests - - name: Build PnetCDF (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - make distclean - ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ - --with-mpi=${GITHUB_WORKSPACE}/MPICH - make -j 8 tests - - name: Print config.log (default configuration) - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files (default configuration) - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make ptests - name: make distcheck run: | cd ${GITHUB_WORKSPACE} diff --git a/.github/workflows/ubuntu_mpich_lustre.yml b/.github/workflows/ubuntu_mpich_lustre.yml new file mode 100644 index 000000000..8c0351c2a --- /dev/null +++ b/.github/workflows/ubuntu_mpich_lustre.yml @@ -0,0 +1,184 @@ +name: ubuntu_mpich (MIMIC_LUSTRE) + +on: + push: + branches: master + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - 'docs/*' + - 'test/test_installed/*' + pull_request: + branches: master + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - 'docs/*' + - 'test/test_installed/*' + +env: + MPICH_VERSION: 4.3.0 + AUTOCONF_VERSION: 2.71 + AUTOMAKE_VERSION: 1.17 + LIBTOOL_VERSION: 2.5.4 + M4_VERSION: 1.4.19 + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - name: Set up dependencies + run: | + sudo apt-get update + # install gfortran + version=12 + sudo add-apt-repository ppa:ubuntu-toolchain-r/test + sudo apt-get update + sudo apt-get install -y gcc-${version} gfortran-${version} + sudo update-alternatives \ + --install /usr/bin/gcc gcc /usr/bin/gcc-${version} 100 \ + --slave /usr/bin/gfortran gfortran /usr/bin/gfortran-${version} \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-${version} + echo "---- gcc/gfortran version ------------------------------" + which gcc + which gfortran + gcc --version + gfortran --version + - name: Clean up git untracked files + run: | + git clean -fx + - name: Build GNU autotools + run: | + export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" + export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz + gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf - + cd m4-${M4_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz + gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf - + cd autoconf-${AUTOCONF_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz + gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf - + cd automake-${AUTOMAKE_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz + gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - + cd libtool-${LIBTOOL_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + - name: Build MPICH + run: | + cd ${GITHUB_WORKSPACE} + echo "Install MPICH ${MPICH_VERSION} in ${GITHUB_WORKSPACE}/MPICH" + rm -rf MPICH ; mkdir MPICH ; cd MPICH + # git clone -q https://github.com/pmodels/mpich.git + # cd mpich + # git submodule update --init + # ./autogen.sh + wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz + gzip -dc mpich-${MPICH_VERSION}.tar.gz | tar -xf - + cd mpich-${MPICH_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/MPICH \ + --silent \ + --enable-romio \ + --with-file-system=ufs \ + --with-device=ch3:sock \ + --enable-fortran \ + CC=gcc FC=gfortran \ + FFLAGS=-fallow-argument-mismatch \ + FCFLAGS=-fallow-argument-mismatch + make -s LIBTOOLFLAGS=--silent V=1 -j 4 install > qout 2>&1 + make -s -j 4 distclean >> qout 2>&1 + - name: Build PnetCDF (MIMIC_LUSTRE) + run: | + cd ${GITHUB_WORKSPACE} + export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" + export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" + m4 --version + autoconf --version + automake --version + libtool --version + autoreconf -i + mkdir -p pnetcdf_output + ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ + --enable-option-checking=fatal \ + pnc_ac_debug=yes \ + --enable-burst_buffering \ + --enable-subfiling \ + --enable-thread-safe \ + --with-pthread \ + --with-mpi=${GITHUB_WORKSPACE}/MPICH \ + TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output \ + MIMIC_LUSTRE=yes + make -j 8 tests + - name: Print config.log + if: ${{ always() }} + run: | + cat ${GITHUB_WORKSPACE}/config.log + - name: make check + run: | + cd ${GITHUB_WORKSPACE} + make check + - name: Print test log files + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + fname=`find src test examples benchmarks -type f -name "*.log"` + for f in $fname ; do \ + bname=`basename $f` ; \ + if test "x$bname" != xconfig.log ; then \ + echo "-------- dump $f ----------------------------" ; \ + cat $f ; \ + fi ; \ + done + - name: make ptests + run: | + cd ${GITHUB_WORKSPACE} + make ptests + - name: make distcheck + run: | + cd ${GITHUB_WORKSPACE} + make -j 8 distcheck DISTCHECK_CONFIGURE_FLAGS="--silent --with-mpi=${GITHUB_WORKSPACE}/MPICH" + - name: make install + run: | + cd ${GITHUB_WORKSPACE} + prefix_path=${GITHUB_WORKSPACE}/pnetcdf_install + echo "---- test make install prefix=${prefix_path}" + make install prefix=${prefix_path} + test/tst_install.sh ${prefix_path} + prefix_path="/pnetcdf_install" + destdir_path=${GITHUB_WORKSPACE}/inst + echo "---- test make install prefix=${prefix_path} DESTDIR=${destdir_path}" + make install prefix=${prefix_path} DESTDIR=${destdir_path} + test/tst_install.sh ${prefix_path} ${destdir_path} + - name: Cleanup + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + make -s distclean + rm -rf ${GITHUB_WORKSPACE}/pnetcdf_output + rm -rf ${GITHUB_WORKSPACE}/MPICH + rm -rf ${GITHUB_WORKSPACE}/pnetcdf_install + rm -rf ${GITHUB_WORKSPACE}/inst + diff --git a/.github/workflows/ubuntu_openmpi.yml b/.github/workflows/ubuntu_openmpi.yml index fb7687ed8..0fb350222 100644 --- a/.github/workflows/ubuntu_openmpi.yml +++ b/.github/workflows/ubuntu_openmpi.yml @@ -123,7 +123,6 @@ jobs: --enable-thread-safe \ --with-pthread \ --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ - MIMIC_LUSTRE=yes \ TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output make -j 8 tests - name: Print config.log @@ -150,41 +149,6 @@ jobs: run: | cd ${GITHUB_WORKSPACE} make ptests - - name: Build PnetCDF (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - make distclean - mkdir -p pnetcdf_output - ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ - --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ - TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output - make -j 8 tests - - name: Print config.log (default configuration) - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files (default configuration) - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make ptests - name: make distcheck run: | cd ${GITHUB_WORKSPACE} diff --git a/.github/workflows/ubuntu_openmpi_adios.yml b/.github/workflows/ubuntu_openmpi_adios.yml index 0ab1925c6..7e956ae6b 100644 --- a/.github/workflows/ubuntu_openmpi_adios.yml +++ b/.github/workflows/ubuntu_openmpi_adios.yml @@ -139,7 +139,6 @@ jobs: --with-pthread \ --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ --with-adios=${GITHUB_WORKSPACE}/ADIOS \ - MIMIC_LUSTRE=yes \ TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output make -j 8 tests - name: Print config.log @@ -166,42 +165,6 @@ jobs: run: | cd ${GITHUB_WORKSPACE} make ptests - - name: Build PnetCDF (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - make distclean - mkdir -p pnetcdf_output - ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ - --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ - --with-adios=${GITHUB_WORKSPACE}/ADIOS \ - TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output - make -j 8 tests - - name: Print config.log (default configuration) - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files (default configuration) - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make ptests - name: make distcheck run: | cd ${GITHUB_WORKSPACE} diff --git a/.github/workflows/ubuntu_openmpi_lustre.yml b/.github/workflows/ubuntu_openmpi_lustre.yml new file mode 100644 index 000000000..4e8dd2765 --- /dev/null +++ b/.github/workflows/ubuntu_openmpi_lustre.yml @@ -0,0 +1,178 @@ +name: ubuntu_openmpi (MIMIC_LUSTRE) + +on: + push: + branches: master + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - 'docs/*' + - 'test/test_installed/*' + pull_request: + branches: master + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - 'docs/*' + - 'test/test_installed/*' + +env: + OPENMPI_VERSION: 5.0.8 + AUTOCONF_VERSION: 2.71 + AUTOMAKE_VERSION: 1.17 + LIBTOOL_VERSION: 2.5.4 + M4_VERSION: 1.4.19 + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - name: Set up dependencies + run: | + sudo apt-get update + # install gfortran + version=12 + sudo add-apt-repository ppa:ubuntu-toolchain-r/test + sudo apt-get update + sudo apt-get install -y gcc-${version} gfortran-${version} + sudo update-alternatives \ + --install /usr/bin/gcc gcc /usr/bin/gcc-${version} 100 \ + --slave /usr/bin/gfortran gfortran /usr/bin/gfortran-${version} \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-${version} + echo "---- gcc/gfortran version ------------------------------" + which gcc + which gfortran + gcc --version + gfortran --version + - name: Clean up git untracked files + run: | + git clean -fx + - name: Build GNU autotools + run: | + export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" + export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz + gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf - + cd m4-${M4_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz + gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf - + cd autoconf-${AUTOCONF_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz + gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf - + cd automake-${AUTOMAKE_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz + gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - + cd libtool-${LIBTOOL_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + - name: Build OPENMPI + run: | + cd ${GITHUB_WORKSPACE} + echo "Install OPENMPI ${OPENMPI_VERSION} in ${GITHUB_WORKSPACE}/OPENMPI" + rm -rf OPENMPI ; mkdir OPENMPI ; cd OPENMPI + VER_MAJOR=${OPENMPI_VERSION%.*} + wget -q https://download.open-mpi.org/release/open-mpi/v${VER_MAJOR}/openmpi-${OPENMPI_VERSION}.tar.gz + gzip -dc openmpi-${OPENMPI_VERSION}.tar.gz | tar -xf - + cd openmpi-${OPENMPI_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/OPENMPI \ + --silent \ + --with-io-romio-flags="--with-file-system=ufs" \ + CC=gcc \ + FC=gfortran \ + FCFLAGS=-fallow-argument-mismatch + make -s LIBTOOLFLAGS=--silent V=1 -j 4 install > qout 2>&1 + make -s -j 4 distclean >> qout 2>&1 + - name: Build PnetCDF (MIMIC_LUSTRE) + run: | + cd ${GITHUB_WORKSPACE} + export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" + export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" + m4 --version + autoconf --version + automake --version + libtool --version + autoreconf -i + mkdir -p pnetcdf_output + ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ + --enable-option-checking=fatal \ + pnc_ac_debug=yes \ + --enable-burst_buffering \ + --enable-subfiling \ + --enable-thread-safe \ + --with-pthread \ + --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ + TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output \ + MIMIC_LUSTRE=yes + make -j 8 tests + - name: Print config.log + if: ${{ always() }} + run: | + cat ${GITHUB_WORKSPACE}/config.log + - name: make check + run: | + cd ${GITHUB_WORKSPACE} + make check + - name: Print test log files + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + fname=`find src test examples benchmarks -type f -name "*.log"` + for f in $fname ; do \ + bname=`basename $f` ; \ + if test "x$bname" != xconfig.log ; then \ + echo "-------- dump $f ----------------------------" ; \ + cat $f ; \ + fi ; \ + done + - name: make ptests + run: | + cd ${GITHUB_WORKSPACE} + make ptests + - name: make distcheck + run: | + cd ${GITHUB_WORKSPACE} + make -j 8 distcheck DISTCHECK_CONFIGURE_FLAGS="--silent --with-mpi=${GITHUB_WORKSPACE}/OPENMPI" + - name: make install + run: | + cd ${GITHUB_WORKSPACE} + prefix_path=${GITHUB_WORKSPACE}/pnetcdf_install + echo "---- test make install prefix=${prefix_path}" + make install prefix=${prefix_path} + test/tst_install.sh ${prefix_path} + prefix_path="/pnetcdf_install" + destdir_path=${GITHUB_WORKSPACE}/inst + echo "---- test make install prefix=${prefix_path} DESTDIR=${destdir_path}" + make install prefix=${prefix_path} DESTDIR=${destdir_path} + test/tst_install.sh ${prefix_path} ${destdir_path} + - name: Cleanup + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + make -s distclean + rm -rf ${GITHUB_WORKSPACE}/pnetcdf_output + rm -rf ${GITHUB_WORKSPACE}/OPENMPI + rm -rf ${GITHUB_WORKSPACE}/pnetcdf_install + rm -rf ${GITHUB_WORKSPACE}/inst + From 9815589c12ec28e2bd65798a790141f98ac00a77 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Fri, 2 Jan 2026 15:47:30 -0800 Subject: [PATCH 5/9] wrf_io.c print hint cb_node_list --- benchmarks/WRF-IO/wrf_io.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/WRF-IO/wrf_io.c b/benchmarks/WRF-IO/wrf_io.c index 85726e16c..f89e5987e 100644 --- a/benchmarks/WRF-IO/wrf_io.c +++ b/benchmarks/WRF-IO/wrf_io.c @@ -1190,7 +1190,7 @@ int wrf_r_benchmark(char *in_file, static int grow_header_benchmark(char *in_file) { - char value[MPI_MAX_INFO_VAL], *attr; + char value[MPI_MAX_INFO_VAL], cb_node_list[MPI_MAX_INFO_VAL], *attr; int i, err=NC_NOERR, nprocs, rank, ncid, ndims, dimid[3]; int varid, unlimdimid, nvars, fix_nvars, rec_nvars, len, flag; double timing, max_t; @@ -1350,6 +1350,12 @@ int grow_header_benchmark(char *in_file) } else nc_data_move_chunk_size = 0; + MPI_Info_get_valuelen(info, "cb_node_list", &len, &flag); + if (flag) + MPI_Info_get(info, "cb_node_list", len+1, cb_node_list, &flag); + else + *cb_node_list = '\0'; + MPI_Info_free(&info); /* close file */ @@ -1384,6 +1390,8 @@ int grow_header_benchmark(char *in_file) printf("Write bandwidth: %.2f MiB/s\n", bw/max_t); printf(" %.2f GiB/s\n", bw/1024.0/max_t); printf("Hint nc_data_move_chunk_size %lld\n", nc_data_move_chunk_size); + if (*cb_node_list != '\0') + printf("Hint cb_node_list = %s\n", cb_node_list); printf("-----------------------------------------------------------\n"); } From 15ba66439ffa50527d5363b2c724ee719bde16bd Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Fri, 2 Jan 2026 15:48:13 -0800 Subject: [PATCH 6/9] move setting hint data_chunk to after file open Value of data_chunk depends on file striping_unit It performs best when set to striping_unit. --- src/drivers/ncmpio/ncmpio_create.c | 26 ++++++++++++++++++++++---- src/drivers/ncmpio/ncmpio_open.c | 25 ++++++++++++++++++++++--- src/drivers/ncmpio/ncmpio_util.c | 4 ++-- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_create.c b/src/drivers/ncmpio/ncmpio_create.c index 7d39b9cd3..a0de9c1e4 100644 --- a/src/drivers/ncmpio/ncmpio_create.c +++ b/src/drivers/ncmpio/ncmpio_create.c @@ -44,7 +44,7 @@ ncmpio_create(MPI_Comm comm, { char *filename, value[MPI_MAX_INFO_VAL + 1], *mpi_name; int rank, nprocs, mpiomode, err, mpireturn, default_format, file_exist=1; - int use_trunc=1, flag; + int use_trunc=1, flag, striping_unit; MPI_File fh=MPI_FILE_NULL; NC *ncp=NULL; @@ -472,10 +472,9 @@ if (rank == 0) printf("%s at %d fstype=%s\n", __func__,__LINE__,(ncp->fstype == if (err != NC_NOERR) DEBUG_FOPEN_ERROR(err) } - /* Copy MPI-IO hints into ncp->mpiinfo */ - ncmpio_hint_set(ncp, ncp->mpiinfo); - fn_exit: + striping_unit = -1; + if (ncp->num_aggrs_per_node > 0) { /* When intra-node aggregation is enabled, it is necessary to make sure * non-aggregators obtain consistent values of file striping hints. @@ -511,8 +510,27 @@ if (rank == 0) printf("%s at %d fstype=%s\n", __func__,__LINE__,(ncp->fstype == sprintf(value, "%d", striping_info[1]); MPI_Info_set(ncp->mpiinfo, "striping_factor", value); } + + striping_unit = striping_info[0]; + } + else { + MPI_Info_get(ncp->mpiinfo, "striping_unit", MPI_MAX_INFO_VAL-1, + value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + striping_unit = (int)strtol(value,NULL,10); + if (errno != 0) striping_unit = -1; + } } + if (ncp->data_chunk == -1) + /* if not set by user hint, nc_data_move_chunk_size */ + ncp->data_chunk = (striping_unit > 0) ? striping_unit + : PNC_DATA_MOVE_CHUNK_SIZE; + + /* Copy MPI-IO hints into ncp->mpiinfo */ + ncmpio_hint_set(ncp, ncp->mpiinfo); + /* if (ncp->rank == 0) { int i, nkeys; diff --git a/src/drivers/ncmpio/ncmpio_open.c b/src/drivers/ncmpio/ncmpio_open.c index fc353aceb..5ca8e2997 100644 --- a/src/drivers/ncmpio/ncmpio_open.c +++ b/src/drivers/ncmpio/ncmpio_open.c @@ -42,6 +42,7 @@ ncmpio_open(MPI_Comm comm, { char *filename, value[MPI_MAX_INFO_VAL + 1], *mpi_name; int i, rank, nprocs, mpiomode, err, status=NC_NOERR, mpireturn, flag; + int striping_unit; MPI_File fh=MPI_FILE_NULL; NC *ncp=NULL; @@ -231,10 +232,9 @@ if (rank == 0) printf("%s at %d fstype=%s\n", __func__,__LINE__,(ncp->fstype == if (err != NC_NOERR) DEBUG_FOPEN_ERROR(err); } - /* Copy MPI-IO hints into ncp->mpiinfo */ - ncmpio_hint_set(ncp, ncp->mpiinfo); - fn_exit: + striping_unit = -1; + if (ncp->num_aggrs_per_node > 0) { /* When intra-node aggregation is enabled, it is necessary to make sure * non-aggregators obtain consistent values of file striping hints. @@ -270,8 +270,27 @@ if (rank == 0) printf("%s at %d fstype=%s\n", __func__,__LINE__,(ncp->fstype == sprintf(value, "%d", striping_info[1]); MPI_Info_set(ncp->mpiinfo, "striping_factor", value); } + + striping_unit = striping_info[0]; + } + else { + MPI_Info_get(ncp->mpiinfo, "striping_unit", MPI_MAX_INFO_VAL-1, + value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + striping_unit = (int)strtol(value,NULL,10); + if (errno != 0) striping_unit = -1; + } } + if (ncp->data_chunk == -1) + /* if not set by user hint, nc_data_move_chunk_size */ + ncp->data_chunk = (striping_unit > 0) ? striping_unit + : PNC_DATA_MOVE_CHUNK_SIZE; + + /* Copy MPI-IO hints into ncp->mpiinfo */ + ncmpio_hint_set(ncp, ncp->mpiinfo); + /* ina_node_list is no longer needed */ if (ncp->ina_node_list != NULL) { NCI_Free(ncp->ina_node_list); diff --git a/src/drivers/ncmpio/ncmpio_util.c b/src/drivers/ncmpio/ncmpio_util.c index ad9fcde61..e73c5a743 100644 --- a/src/drivers/ncmpio/ncmpio_util.c +++ b/src/drivers/ncmpio/ncmpio_util.c @@ -45,7 +45,7 @@ void ncmpio_hint_extract(NC *ncp, ncp->hdr_chunk = PNC_HDR_READ_CHUNK_SIZE; /* chunk size for moving variables to higher offsets */ - ncp->data_chunk = PNC_DATA_MOVE_CHUNK_SIZE; + ncp->data_chunk = -1; /* buffer to pack noncontiguous user buffers when calling wait() */ ncp->ibuf_size = PNC_DEFAULT_IBUF_SIZE; @@ -256,7 +256,7 @@ void ncmpio_hint_extract(NC *ncp, llval = strtoll(value, NULL, 10); if (errno == 0) { if (llval < 0) - ncp->data_chunk = PNC_DATA_MOVE_CHUNK_SIZE; + ncp->data_chunk = -1; else if (llval > NC_MAX_INT) /* limit to NC_MAX_INT */ ncp->data_chunk = NC_MAX_INT; else From 823853ab18cceb71a7a36913ebdf554b38c24cb5 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Fri, 2 Jan 2026 16:42:27 -0800 Subject: [PATCH 7/9] Improve data section movement * use file striping unit size as moving chunk size by default. * align write offsets with the striping units to coincide the Lustre collective write file domain alignment --- src/drivers/ncmpio/ncmpio_enddef.c | 57 +++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_enddef.c b/src/drivers/ncmpio/ncmpio_enddef.c index 63de09ff9..13af162d4 100644 --- a/src/drivers/ncmpio/ncmpio_enddef.c +++ b/src/drivers/ncmpio/ncmpio_enddef.c @@ -164,9 +164,9 @@ move_file_block(NC *ncp, MPI_Offset from, /* source starting file offset */ MPI_Offset nbytes) /* amount to be moved */ { - int rank, nprocs, status=NC_NOERR, do_coll; + int rank, align_rank, nprocs, status=NC_NOERR, do_coll; void *buf; - MPI_Offset last_block, mv_amnt, p_units; + MPI_Offset last_block, mv_amnt, p_units, end_off, end_block; MPI_Offset off_last, off_from, off_to, rlen, wlen; MPI_Comm comm; PNCIO_View buf_view; @@ -181,6 +181,10 @@ move_file_block(NC *ncp, rank = (ncp->ina_comm == MPI_COMM_NULL) ? ncp->rank : ncp->ina_rank; nprocs = (ncp->ina_comm == MPI_COMM_NULL) ? ncp->nprocs : ncp->ina_nprocs; + /* align file access for all ranks */ + align_rank = rank + (to / ncp->data_chunk); + align_rank %= nprocs; + /* Use MPI collective I/O subroutines to move data, only if nproc > 1 and * MPI-IO hint "romio_no_indep_rw" is set to true. Otherwise, use MPI * independent I/O subroutines, as the data partitioned among processes are @@ -207,25 +211,34 @@ move_file_block(NC *ncp, /* movement must start from the last p_units toward to the 1st */ p_units = (MPI_Offset)ncp->data_chunk * nprocs; - off_last = nbytes / p_units; - last_block = nbytes % p_units; - if (last_block == 0) { - off_last--; - last_block = p_units; - } - /* mv_amnt is the amount moved by this rank at the first round */ - if (rank < last_block / ncp->data_chunk) + end_off = to + nbytes; + end_block = end_off % p_units; + off_last = end_off - end_block; + + /* align file writes for all ranks (reads will not be aligned) */ + if (align_rank < end_block / ncp->data_chunk) mv_amnt = ncp->data_chunk; - else if (last_block % ncp->data_chunk > 0 && - rank == last_block / ncp->data_chunk) - mv_amnt = last_block % ncp->data_chunk; + else if (end_block % ncp->data_chunk > 0 && + align_rank == end_block / ncp->data_chunk) + mv_amnt = end_block % ncp->data_chunk; else mv_amnt = 0; - off_last *= p_units; - off_last += (MPI_Offset)rank * ncp->data_chunk; - off_from = from + off_last; - off_to = to + off_last; + /* set the 1st read-write pair */ + off_to = off_last + (MPI_Offset)align_rank * ncp->data_chunk; + off_from = off_to - (to - from); + + if (off_from < from) + off_from = from; + + if (off_to < to) { + mv_amnt -= to - off_to; + if (mv_amnt < 0) mv_amnt = 0; + off_to = to; + } + + /* pad the remaining of last p_units */ + nbytes += p_units - end_block; while (nbytes > 0) { buf_view.size = mv_amnt; @@ -259,9 +272,19 @@ move_file_block(NC *ncp, /* mv_amnt becomes ncp->data_chunk in the 2nd and later rounds */ mv_amnt = ncp->data_chunk; + + /* special treatment for the 1st p_units */ + if (off_to < to) { + mv_amnt -= to - off_to; + if (mv_amnt < 0) mv_amnt = 0; + off_to = to; + } + if (off_from < from) + off_from = from; } NCI_Free(buf); + return status; } #endif From abee10cec646bd9c71a3af256c842c92899e0379 Mon Sep 17 00:00:00 2001 From: wkliao Date: Sat, 3 Jan 2026 16:01:13 -0600 Subject: [PATCH 8/9] parallel_run.sh setenv MIMIC_STRIPE_SIZE to 1048576 --- examples/parallel_run.sh | 100 +++++++++++++++++++++++---------------- test/parallel_run.sh | 16 +++++++ 2 files changed, 74 insertions(+), 42 deletions(-) diff --git a/examples/parallel_run.sh b/examples/parallel_run.sh index 5a6c76f82..637342d77 100755 --- a/examples/parallel_run.sh +++ b/examples/parallel_run.sh @@ -7,6 +7,41 @@ # Exit immediately if a command exits with a non-zero status. set -e +DRY_RUN=no +VERBOSE=no + +exe_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + + cmd=`basename $1` + if test "x$MIMIC_LUSTRE" = x1 && test "x$cmd" = xncmpidiff ; then + # echo "export MIMIC_STRIPE_SIZE=1048576" + export MIMIC_STRIPE_SIZE=1048576 + fi + + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi + + if test "x$MIMIC_LUSTRE" = x1 && test "x$cmd" = xncmpidiff ; then + # echo "unset MIMIC_STRIPE_SIZE" + unset MIMIC_STRIPE_SIZE + fi +} + +seq_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} + VALIDATOR=../../src/utils/ncvalidator/ncvalidator NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff @@ -54,6 +89,7 @@ for i in ${check_PROGRAMS} ; do DRIVER_OUT_FILE="${OUT_PREFIX}.pncio" driver_hint="PNCIO" fi + for intra_aggr in 0 1 ; do if test "$intra_aggr" = 1 ; then INA_HINTS="nc_num_aggrs_per_node=2" @@ -90,7 +126,9 @@ for i in ${check_PROGRAMS} ; do export PNETCDF_HINTS="$PNETCDF_HINTS" export PNETCDF_SAFE_MODE=$j - # echo "${LINENO}: PNETCDF_SAFE_MODE=$PNETCDF_SAFE_MODE PNETCDF_HINTS=$PNETCDF_HINTS" + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line ${LINENO}: PNETCDF_SAFE_MODE=$PNETCDF_SAFE_MODE PNETCDF_HINTS=$PNETCDF_HINTS" + fi TEST_OPTS="$safe_hint $driver_hint $ina_hint" @@ -102,27 +140,22 @@ for i in ${check_PROGRAMS} ; do if test "$i" = pthread ; then # each MPI process created 6 threads - # echo "${LINENO}: ${MPIRUN} ./$i $CMD_OPT ${OUT_FILE}.nc" - ${MPIRUN} ./$i $CMD_OPT ${OUT_FILE}.nc + exe_cmd ./$i $CMD_OPT ${OUT_FILE}.nc for k in `seq 0 ${NTHREADS}` ; do - # echo "${LINENO}: --- validating file ${OUT_FILE}.nc.$k" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.nc.$k + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc.$k done if test $? = 0 ; then printf "PASS: %-3s nprocs=$1 %-${fixed_length}s -------- $i\n" $2 "$TEST_OPTS" fi continue elif test "$i" = put_vara ; then - # echo "${LINENO}: ${MPIRUN} ./$i $CMD_OPT ${OUT_FILE}.nc" - ${MPIRUN} ./$i $CMD_OPT ${OUT_FILE}.nc - # echo "${LINENO}: --- validating file ${OUT_FILE}.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.nc + exe_cmd ./$i $CMD_OPT ${OUT_FILE}.nc + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc if test $? = 0 ; then printf "PASS: %-3s nprocs=$1 %-${fixed_length}s -------- $i\n" $2 "$TEST_OPTS" fi - # echo "${LINENO}: ${MPIRUN} ./get_vara $CMD_OPT ${OUT_FILE}.nc" - ${MPIRUN} ./get_vara $CMD_OPT ${OUT_FILE}.nc + exe_cmd ./get_vara $CMD_OPT ${OUT_FILE}.nc if test $? = 0 ; then printf "PASS: %-3s nprocs=$1 %-${fixed_length}s -------- get_vara\n" $2 "$TEST_OPTS" fi @@ -130,56 +163,45 @@ for i in ${check_PROGRAMS} ; do continue elif test "$i" = create_from_cdl ; then # create_from_cdl reads a CDL header file - # echo "${LINENO}: ${MPIRUN} ./$i $CMD_OPT -o ${OUT_FILE}.nc $IN_FILE" - ${MPIRUN} ./$i $CMD_OPT -o ${OUT_FILE}.nc $IN_FILE + exe_cmd ./$i $CMD_OPT -o ${OUT_FILE}.nc $IN_FILE if test $? = 0 ; then printf "PASS: %-3s nprocs=$1 %-${fixed_length}s -------- $i\n" $2 "$TEST_OPTS" fi else - # echo "${LINENO}: ${MPIRUN} ./$i $CMD_OPT ${OUT_FILE}.nc" - ${MPIRUN} ./$i $CMD_OPT ${OUT_FILE}.nc + exe_cmd ./$i $CMD_OPT ${OUT_FILE}.nc if test $? = 0 ; then printf "PASS: %-3s nprocs=$1 %-${fixed_length}s -------- $i\n" $2 "$TEST_OPTS" fi fi - # echo "${LINENO}:--- validating file ${OUT_FILE}.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.nc + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - # echo "${LINENO}: ---- test burst buffering feature" saved_PNETCDF_HINTS=${PNETCDF_HINTS} export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - # echo "${LINENO}:--- ./$i $CMD_OPT ${OUT_FILE}.bb.nc" if test "$i" = create_from_cdl ; then # create_from_cdl reads a CDL header file - # echo "${LINENO}: ${MPIRUN} ./$i -q -o ${OUT_FILE}.bb.nc $IN_FILE" - ${MPIRUN} ./$i -q -o ${OUT_FILE}.bb.nc $IN_FILE + exe_cmd ./$i -q -o ${OUT_FILE}.bb.nc $IN_FILE else - # echo "${LINENO}: ${MPIRUN} ./$i $CMD_OPT ${OUT_FILE}.bb.nc" - ${MPIRUN} ./$i $CMD_OPT ${OUT_FILE}.bb.nc + exe_cmd ./$i $CMD_OPT ${OUT_FILE}.bb.nc fi if test $? = 0 ; then printf "PASS: %-3s nprocs=$1 %-${fixed_length}s -------- $i\n" $2 "$TEST_OPTS BB" fi export PNETCDF_HINTS=${saved_PNETCDF_HINTS} - # echo "${LINENO}: --- validating file ${OUT_FILE}.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${OUT_FILE}.bb.nc + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.bb.nc # compare file header only for large file tests DIFF_OPT="-q" if test "$i" = create_from_cdl ; then DIFF_OPT+=" -h" fi - # echo "${LINENO}: --- ncmpidiff $DIFF_OPT $OUT_FILE.nc $OUT_FILE.bb.nc ---" - ${MPIRUN} ${NCMPIDIFF} $DIFF_OPT $OUT_FILE.nc $OUT_FILE.bb.nc + exe_cmd ${NCMPIDIFF} $DIFF_OPT $OUT_FILE.nc $OUT_FILE.bb.nc fi if test "x${ENABLE_NETCDF4}" = x1 ; then - # echo "${LINENO}: test netCDF-4 feature" - # echo "${LINENO}: ${MPIRUN} ./$i ${OUT_FILE}.nc4 4" - ${MPIRUN} ./$i ${OUT_FILE}.nc4 4 + exe_cmd ./$i ${OUT_FILE}.nc4 4 if test $? = 0 ; then printf "PASS: %-3s nprocs=$1 %-${fixed_length}s -------- $i\n" $2 "$TEST_OPTS" fi @@ -202,20 +224,14 @@ for i in ${check_PROGRAMS} ; do fi if test "$i" = pthread ; then for j in `seq 0 ${NTHREADS}` ; do - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.mpio.ina.nc.$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.mpio.ina.nc.$j - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.nc.$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.nc.$j - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.mpio.ina.nc.$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.pncio.nc.$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc.$j $OUT_PREFIX.pncio.ina.nc.$j done else - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc - # echo "${LINENO}: --- ncmpidiff $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc ---" - $MPIRUN $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.pncio.nc + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.pncio.nc $OUT_PREFIX.pncio.ina.nc fi done # no_indep_rw diff --git a/test/parallel_run.sh b/test/parallel_run.sh index f5e180a6b..c1d4fcde9 100755 --- a/test/parallel_run.sh +++ b/test/parallel_run.sh @@ -12,12 +12,28 @@ VERBOSE=no exe_cmd() { local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + + saved_PNETCDF_HINTS= + cmd=`basename $1` + if test "x$MIMIC_LUSTRE" = x1 && test "x$cmd" = xncmpidiff ; then + # echo "export MIMIC_STRIPE_SIZE=1048576" + export MIMIC_STRIPE_SIZE=1048576 + saved_PNETCDF_HINTS=$PNETCDF_HINTS + unset PNETCDF_HINTS + fi + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then echo "Line $lineno CMD: $MPIRUN $@" fi if test "x$DRY_RUN" = xno ; then $MPIRUN $@ fi + + if test "x$MIMIC_LUSTRE" = x1 && test "x$cmd" = xncmpidiff ; then + # echo "unset MIMIC_STRIPE_SIZE" + unset MIMIC_STRIPE_SIZE + export PNETCDF_HINTS=$saved_PNETCDF_HINTS + fi } seq_cmd() { From 0760380992c5973e1d5873198ef4d2b9a4c97495 Mon Sep 17 00:00:00 2001 From: wkliao Date: Sat, 3 Jan 2026 16:11:20 -0600 Subject: [PATCH 9/9] Github actions: increase timeout to 120 minutes --- .github/workflows/mac_mpich.yml | 2 +- .github/workflows/mac_mpich_lustre.yml | 2 +- .github/workflows/mac_openmpi.yml | 2 +- .github/workflows/mac_openmpi_lustre.yml | 2 +- .github/workflows/ubuntu_mpich.yml | 2 +- .github/workflows/ubuntu_mpich_lustre.yml | 2 +- .github/workflows/ubuntu_openmpi.yml | 2 +- .github/workflows/ubuntu_openmpi_adios.yml | 2 +- .github/workflows/ubuntu_openmpi_lustre.yml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mac_mpich.yml b/.github/workflows/mac_mpich.yml index c50885758..bd52d5ac8 100644 --- a/.github/workflows/mac_mpich.yml +++ b/.github/workflows/mac_mpich.yml @@ -32,7 +32,7 @@ env: jobs: build: runs-on: macos-latest - timeout-minutes: 90 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 - name: Set up dependencies diff --git a/.github/workflows/mac_mpich_lustre.yml b/.github/workflows/mac_mpich_lustre.yml index c2f7fe8b5..444449a4a 100644 --- a/.github/workflows/mac_mpich_lustre.yml +++ b/.github/workflows/mac_mpich_lustre.yml @@ -32,7 +32,7 @@ env: jobs: build: runs-on: macos-latest - timeout-minutes: 90 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 - name: Set up dependencies diff --git a/.github/workflows/mac_openmpi.yml b/.github/workflows/mac_openmpi.yml index 158af438b..b0fdfd66d 100644 --- a/.github/workflows/mac_openmpi.yml +++ b/.github/workflows/mac_openmpi.yml @@ -32,7 +32,7 @@ env: jobs: build: runs-on: macos-latest - timeout-minutes: 90 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 - name: Set up dependencies diff --git a/.github/workflows/mac_openmpi_lustre.yml b/.github/workflows/mac_openmpi_lustre.yml index 303a22a66..f90fc8b2b 100644 --- a/.github/workflows/mac_openmpi_lustre.yml +++ b/.github/workflows/mac_openmpi_lustre.yml @@ -32,7 +32,7 @@ env: jobs: build: runs-on: macos-latest - timeout-minutes: 90 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 - name: Set up dependencies diff --git a/.github/workflows/ubuntu_mpich.yml b/.github/workflows/ubuntu_mpich.yml index 6126e78ee..5a41e36b7 100644 --- a/.github/workflows/ubuntu_mpich.yml +++ b/.github/workflows/ubuntu_mpich.yml @@ -28,7 +28,7 @@ env: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 - name: Set up dependencies diff --git a/.github/workflows/ubuntu_mpich_lustre.yml b/.github/workflows/ubuntu_mpich_lustre.yml index 8c0351c2a..6472279d4 100644 --- a/.github/workflows/ubuntu_mpich_lustre.yml +++ b/.github/workflows/ubuntu_mpich_lustre.yml @@ -28,7 +28,7 @@ env: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 - name: Set up dependencies diff --git a/.github/workflows/ubuntu_openmpi.yml b/.github/workflows/ubuntu_openmpi.yml index 0fb350222..b12d389b9 100644 --- a/.github/workflows/ubuntu_openmpi.yml +++ b/.github/workflows/ubuntu_openmpi.yml @@ -28,7 +28,7 @@ env: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 - name: Set up dependencies diff --git a/.github/workflows/ubuntu_openmpi_adios.yml b/.github/workflows/ubuntu_openmpi_adios.yml index 7e956ae6b..71b735e40 100644 --- a/.github/workflows/ubuntu_openmpi_adios.yml +++ b/.github/workflows/ubuntu_openmpi_adios.yml @@ -29,7 +29,7 @@ env: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 - name: Set up dependencies diff --git a/.github/workflows/ubuntu_openmpi_lustre.yml b/.github/workflows/ubuntu_openmpi_lustre.yml index 4e8dd2765..76faecc52 100644 --- a/.github/workflows/ubuntu_openmpi_lustre.yml +++ b/.github/workflows/ubuntu_openmpi_lustre.yml @@ -28,7 +28,7 @@ env: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 - name: Set up dependencies