Enable ncmpi_inq_header_size() to be called in define mode.

sneak_peek.md

@@ -26,7 +26,14 @@ This is essentially a placeholder for the next release note ...
   + none
 
 * API semantics updates
-  + none
+  + API ncmpi_inq_header_size() now can be called in the define mode. This API
+    returns the file header size with metadata defined by the time of the call.
+    This information can be helpful to pick proper values for arguments
+    h_minfree, v_align, v_minfree, r_align when calling ncmpi__enddef() to
+    allocate a sufficiently large free space for file header extent and
+    variable data sections to grow without moving data already stored in the
+    file, i.e. when adding new variables, dimensions, or attributes.
+    See [PR #201](https://github.com/Parallel-NetCDF/PnetCDF/pull/201).
 
 * New error code precedence
   + none

src/drivers/ncmpio/ncmpio_file_misc.c

@@ -423,7 +423,17 @@ ncmpio_inq_misc(void       *ncdp,
 
     if (recsize != NULL) *recsize = ncp->recsize;
 
-    if (header_size != NULL) *header_size = ncp->xsz;
+    if (header_size != NULL) {
+        if (NC_indef(ncp))
+            /* When called in define mode, calculate and return the current
+             * header size. Cannot do the same for header extent, as the empty
+             * space depends on arguments h_minfree and v_align of
+             * ncmpi__enddef().
+             */
+            *header_size = ncmpio_hdr_len_NC(ncp);
+        else
+            *header_size = ncp->xsz;
+    }
 
     if (header_extent != NULL) *header_extent = ncp->begin_var;
 

test/testcases/Makefile.am

@@ -101,7 +101,8 @@ TESTPROGRAMS = file_create_open \
                tst_grow_header \
                tst_varn_var1 \
                tst_multi_redefine \
-               tst_grow_data
+               tst_grow_data \
+               tst_inq_header_size
 
 M4_SRCS      = put_all_kinds.m4 \
                erange_fill.m4 \

test/testcases/tst_inq_header_size.c

@@ -0,0 +1,281 @@
+/*
+ *  Copyright (C) 2025, Northwestern University and Argonne National Laboratory
+ *  See COPYRIGHT notice in top-level directory.
+ */
+
+/*
+ * This program tests a call to ncmpi_inq_header_size() when in the define
+ * mode, which should calculate and return the latest file header size. This
+ * can be useful for application users to decide how much free space to be
+ * preserved in the file header section, i.e. by setting argument h_minfree
+ * and/or v_align when calling ncmpi__enddef().
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h> /* strcasecmp() */
+#include <libgen.h> /* basename() */
+#include <mpi.h>
+#include <pnetcdf.h>
+
+#include <testutils.h>
+
+static int debug;
+
+static int
+tst_fmt(char *filename, int cmode)
+{
+    char *str;
+    int  err, nerrs=0, rank, ncid, dimids[2], varid, int_buf;
+    float flt_buf;
+    double *dbl_buf;
+    MPI_Offset old_h_size, old_h_extent, new_h_size, new_h_extent;
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+    /* create a file */
+    cmode |= NC_CLOBBER;
+    err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid);
+    CHECK_ERR
+
+    flt_buf = 1.234;
+    err = ncmpi_put_att(ncid, NC_GLOBAL, "_FillValue", NC_FLOAT, 1, &flt_buf);
+    CHECK_ERR
+
+    err = ncmpi_def_dim(ncid, "X", 10, &dimids[0]); CHECK_ERR
+    CHECK_ERR
+
+    err = ncmpi_def_var(ncid, "int_var", NC_INT, 1, dimids, &varid);
+    CHECK_ERR
+
+    err = ncmpi_put_att(ncid, varid, "_FillValue", NC_FLOAT, 1, &flt_buf);
+    EXP_ERR(NC_EBADTYPE)
+
+    int_buf = 5678;
+    err = ncmpi_put_att(ncid, varid, "_FillValue", NC_INT, 1, &int_buf);
+    CHECK_ERR
+
+    err = ncmpi_def_var(ncid, "dbl_var", NC_DOUBLE, 1, dimids, &varid);
+    CHECK_ERR
+
+    err = ncmpi_def_var(ncid, "short_var", NC_SHORT, 1, dimids, &varid);
+    CHECK_ERR
+
+    err = ncmpi_set_fill(ncid, NC_FILL, NULL); CHECK_ERR
+
+    err = ncmpi_inq_header_size(ncid, &old_h_size); CHECK_ERR
+
+    err = ncmpi_inq_header_extent(ncid, &old_h_extent); CHECK_ERR
+
+    if (debug && rank == 0)
+        printf("%s at %d: header size=%lld extent=%lld\n", __FILE__,__LINE__,
+               old_h_size, old_h_extent);
+
+    if (old_h_extent != 0) {
+        printf("Error at %d: expect file extent size to be 0 but got %lld\n",
+               __LINE__, old_h_extent);
+        nerrs++;
+        goto err_out;
+    }
+
+    err = ncmpi_enddef(ncid); CHECK_ERR
+
+    err = ncmpi_inq_header_size(ncid, &new_h_size); CHECK_ERR
+
+    err = ncmpi_inq_header_extent(ncid, &new_h_extent); CHECK_ERR
+
+    if (debug && rank == 0)
+        printf("%s at %d: header size=%lld extent=%lld\n", __FILE__,__LINE__,
+               new_h_size, new_h_extent);
+
+    if (new_h_size != old_h_size) {
+        printf("Error at %d: expect file header size %lld but got %lld\n",
+               __LINE__, old_h_size, new_h_size);
+        nerrs++;
+        goto err_out;
+    }
+
+    if (new_h_extent <= old_h_extent) {
+        printf("Error at %d: expect file extent size > %lld but got %lld\n",
+               __LINE__, old_h_extent, new_h_extent);
+        nerrs++;
+        goto err_out;
+    }
+
+    err = ncmpi_close(ncid); CHECK_ERR
+
+    old_h_size = new_h_size;
+    old_h_extent = new_h_extent;
+
+    /* open the file */
+    err = ncmpi_open(MPI_COMM_WORLD, filename, NC_WRITE, MPI_INFO_NULL, &ncid);
+    CHECK_ERR
+
+    err = ncmpi_inq_header_size(ncid, &new_h_size); CHECK_ERR
+
+    err = ncmpi_inq_header_extent(ncid, &new_h_extent); CHECK_ERR
+
+    if (debug && rank == 0)
+        printf("%s at %d: header size=%lld extent=%lld\n", __FILE__,__LINE__,
+               new_h_size, new_h_extent);
+
+    if (new_h_size != old_h_size) {
+        printf("Error at %d: expect file header size %lld but got %lld\n",
+               __LINE__, old_h_size, new_h_size);
+        nerrs++;
+        goto err_out;
+    }
+
+    if (new_h_extent != old_h_extent) {
+        printf("Error at %d: expect file extent size %lld but got %lld\n",
+               __LINE__, old_h_extent, new_h_extent);
+        nerrs++;
+        goto err_out;
+    }
+
+    old_h_size = new_h_size;
+    old_h_extent = new_h_extent;
+
+    /* enter define mode and add new a dimension and a variable */
+    err = ncmpi_redef(ncid); CHECK_ERR
+
+    str = "new global attribute of text data type";
+    err = ncmpi_put_att_text(ncid, NC_GLOBAL, "global_attr", strlen(str), str);
+    CHECK_ERR
+
+    err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimids[0]); CHECK_ERR
+    CHECK_ERR
+
+    err = ncmpi_def_dim(ncid, "Y", 10, &dimids[1]); CHECK_ERR
+    CHECK_ERR
+
+    err = ncmpi_def_var(ncid, "new_int_var", NC_INT, 2, dimids, &varid);
+    CHECK_ERR
+
+    dbl_buf = (double*) calloc(16, sizeof(double));
+    err = ncmpi_put_att_double(ncid, varid, "attr", NC_DOUBLE, 16, dbl_buf);
+    CHECK_ERR
+    free(dbl_buf);
+
+    err = ncmpi_inq_header_size(ncid, &new_h_size); CHECK_ERR
+
+    err = ncmpi_inq_header_extent(ncid, &new_h_extent); CHECK_ERR
+
+    if (debug && rank == 0)
+        printf("%s at %d: header size=%lld extent=%lld\n", __FILE__,__LINE__,
+               new_h_size, new_h_extent);
+
+    if (new_h_size <= old_h_size) {
+        printf("Error at %d: expect file header size > %lld but got %lld\n",
+               __LINE__, old_h_size, new_h_size);
+        nerrs++;
+        goto err_out;
+    }
+
+    if (new_h_extent != old_h_extent) {
+        printf("Error at %d: expect file extent size %lld but got %lld\n",
+               __LINE__, old_h_extent, new_h_extent);
+        nerrs++;
+        goto err_out;
+    }
+
+    if (new_h_size > old_h_extent)
+        /* header size grows beyond the current file extent size */
+        err = ncmpi__enddef(ncid, 0, 512, 0, 0);
+    else
+        err = ncmpi_enddef(ncid);
+    CHECK_ERR
+
+    old_h_size = new_h_size;
+    old_h_extent = new_h_extent;
+
+    err = ncmpi_inq_header_size(ncid, &new_h_size); CHECK_ERR
+
+    err = ncmpi_inq_header_extent(ncid, &new_h_extent); CHECK_ERR
+
+    if (debug && rank == 0)
+        printf("%s at %d: header size=%lld extent=%lld\n", __FILE__,__LINE__,
+               new_h_size, new_h_extent);
+
+    if (new_h_size != old_h_size) {
+        printf("Error at %d: expect file header size %lld but got %lld\n",
+               __LINE__, old_h_size, new_h_size);
+        nerrs++;
+        goto err_out;
+    }
+
+    if (new_h_extent < old_h_extent) {
+        printf("Error at %d: expect file extent size >= %lld but got %lld\n",
+               __LINE__, old_h_extent, new_h_extent);
+        nerrs++;
+        goto err_out;
+    }
+
+    err = ncmpi_close(ncid); CHECK_ERR
+
+err_out:
+    return nerrs;
+}
+
+int main(int argc, char **argv) {
+    char filename[256], *hint_value;
+    int  err, nerrs=0, rank, bb_enabled=0;
+
+    MPI_Init(&argc, &argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+    debug = 0;
+
+    if (argc > 2) {
+        if (!rank) printf("Usage: %s [filename]\n",argv[0]);
+        MPI_Finalize();
+        return 1;
+    }
+    if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
+    else           strcpy(filename, "testfile.nc");
+    MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD);
+
+    if (rank == 0) {
+        char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
+        sprintf(cmd_str, "*** TESTING C   %s for get header size in define mode", basename(argv[0]));
+        printf("%-66s ------ ", cmd_str); fflush(stdout);
+        free(cmd_str);
+    }
+
+    /* check whether burst buffering is enabled */
+    if (inq_env_hint("nc_burst_buf", &hint_value)) {
+        if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1;
+        free(hint_value);
+    }
+
+    nerrs += tst_fmt(filename, 0);
+    nerrs += tst_fmt(filename, NC_64BIT_OFFSET);
+    if (!bb_enabled) {
+#ifdef ENABLE_NETCDF4
+        nerrs += tst_fmt(filename, NC_NETCDF4);
+        nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL);
+#endif
+    }
+    nerrs += tst_fmt(filename, NC_64BIT_DATA);
+
+    /* check if PnetCDF freed all internal malloc */
+    MPI_Offset malloc_size, sum_size;
+    err = ncmpi_inq_malloc_size(&malloc_size);
+    if (err == NC_NOERR) {
+        MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
+        if (rank == 0 && sum_size > 0)
+            printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n",
+                   sum_size);
+        if (malloc_size > 0) ncmpi_inq_malloc_list();
+    }
+
+    MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+    if (rank == 0) {
+        if (nerrs) printf(FAIL_STR,nerrs);
+        else       printf(PASS_STR);
+    }
+
+    MPI_Finalize();
+    return (nerrs > 0);
+}