Add NetCDF4/HDF5 support, including support for compressed NetCDF trajectories.

.github/workflows/merge-gate.yml

@@ -63,16 +63,6 @@ jobs:
           mkdir -p include && mv AmberTools/src/sander/sander.h include
           mv lib include $HOME
 
-          curl -OL ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.6.1.tar.gz
-          tar -zxf netcdf-4.6.1.tar.gz
-          cd netcdf-4.6.1
-          ./configure --disable-netcdf-4 --disable-dap --disable-doxygen --prefix=$HOME
-          make -j2
-          make install
-          cd ..
-          export PATH=$HOME/bin:$PATH
-          export LD_LIBRARY_PATH=$HOME/lib:${LD_LIBRARY_PATH}
-
           if [ $USE_OPENMP = "yes" ]; then
             export OPT="openmp"
             export OMP_NUM_THREADS=4
@@ -105,6 +95,16 @@ jobs:
           fi
 
           if [ "$USE_CMAKE" = "1" ]; then
+            curl -OL ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.6.1.tar.gz
+            tar -zxf netcdf-4.6.1.tar.gz
+            cd netcdf-4.6.1
+            ./configure --disable-netcdf-4 --disable-dap --disable-doxygen --prefix=$HOME
+            make -j2
+            make install
+            cd ..
+            export PATH=$HOME/bin:$PATH
+            export LD_LIBRARY_PATH=$HOME/lib:${LD_LIBRARY_PATH}
+
             mkdir build install
             installdir=$(pwd)
             cd build
@@ -117,7 +117,8 @@ jobs:
             export PATH=$installdir/bin:$PATH
           else
             export LD_LIBRARY_PATH=$HOME/lib:${LD_LIBRARY_PATH}
-            ./configure --with-netcdf=$HOME ${BUILD_FLAGS} ${COMPILER}
+            ./configure --buildlibs ${BUILD_FLAGS} ${COMPILER}
+            source cpptraj.sh
             make -j2 install
           fi
           cd test && make $TEST_TYPE

appveyor.yml

@@ -1,7 +1,7 @@
 platform:
   - x64
 
-image: Visual Studio 2019
+image: Previous Visual Studio 2019
 
 clone_folder: c:\projects\cpptraj
 

configure

@@ -134,6 +134,12 @@ NETCDF_SRCTAR='netcdf-4.6.1.tar.gz'
 NETCDF_SRCDIR='netcdf-4.6.1'
 NETCDF_URL="ftp://ftp.unidata.ucar.edu/pub/netcdf/$NETCDF_SRCTAR"
 NETCDF_OPTS=" --disable-netcdf-4 --disable-dap $windows_hostflag --disable-shared --disable-doxygen"
+NETCDF4_OPTS=" --disable-dap $windows_hostflag --disable-shared --disable-doxygen"
+
+HDF5_SRCTAR='hdf5-1_10_9.tar.gz'
+HDF5_SRCDIR='hdf5-hdf5-1_10_9'
+HDF5_URL="https://github.com/HDFGroup/hdf5/archive/refs/tags/$HDF5_SRCTAR"
+HDF5_OPTS=''
 
 BZIP2_SRCTAR='bzip2-latest.tar.gz'
 BZIP2_SRCDIR=''
@@ -222,13 +228,14 @@ LBLAS=7
 LFFTW3=8
 LREADLINE=9
 LXDRFILE=10
-LSANDER=11
-LTIMER=12
-LCUDA=13
-LHIP=14
-LOPENMM=15
+LTIMER=11
+LCUDA=12
+LHIP=13
+LOPENMM=14
+LHDF5=15
+LSANDER=16  # Make sure this comes last to avoid pulling in unnecessary symbols
 # Total number of external libraries
-NLIB=16
+NLIB=17
 
 # LIB_STAT = Library status:
 #   off       : Do not use library.
@@ -336,7 +343,7 @@ LIB_FLAG[$LSANDER]='-lsander'
 LIB_STTC[$LSANDER]=''
 LIB_D_ON[$LSANDER]='-DUSE_SANDERLIB'
 LIB_DOFF[$LSANDER]=''
-LIB_TYPE[$LSANDER]='cpp'
+LIB_TYPE[$LSANDER]='ld'
 
 LIB_STAT[$LTIMER]='off'
 LIB_CKEY[$LTIMER]='timer'
@@ -372,6 +379,16 @@ LIB_D_ON[$LTNGFILE]='-DHAS_TNGFILE'
 LIB_DOFF[$LTNGFILE]=''
 LIB_TYPE[$LTNGFILE]='cpp'
 
+LIB_STAT[$LHDF5]='enabled'
+LIB_CKEY[$LHDF5]='hdf5'
+LIB_HOME[$LHDF5]=''
+LIB_FLAG[$LHDF5]=''
+LIB_STTC[$LHDF5]='libhdf5.a'
+LIB_D_ON[$LHDF5]='-DHAS_HDF5'
+LIB_DOFF[$LHDF5]=''
+LIB_LINK[$LHDF5]='dynamic'
+LIB_TYPE[$LHDF5]='ld'
+
 LIB_STAT[$LOPENMM]='off'
 LIB_CKEY[$LOPENMM]='openmm'
 LIB_HOME[$LOPENMM]=''
@@ -531,6 +548,8 @@ TestProgram() {
     else
       COMPILE_ERROR_LINE=$COMPILELINE
       COMPILE_ERROR_MESSAGE=`cat $COMPERR`
+      #echo "DEBUG: Error: $COMPILE_ERROR_LINE"
+      #echo "DEBUG: Error: $COMPILE_ERROR_MESSAGE"
       rm $COMPERR $COMPOUT
       return 1
     fi
@@ -641,6 +660,82 @@ EOF
   fi
 }
 
+TestHdf5() {
+  # First, check if NetCDF has HDF5 support baked in.
+  cat > testp.cpp <<EOF
+#include <cstdio>
+#include "netcdf.h"
+void unused() {
+  int ncid, varid;
+  int dims[1];
+  // Create a netcdf4 file
+  nc_create("foo.nc", NC_NETCDF4, &ncid);
+  // Create a dimension
+  dims[1] = 10;
+  nc_def_var(ncid, "MyVar", NC_DOUBLE, 1, dims, &varid);
+  // Set deflate
+  int err = nc_def_var_deflate(ncid, varid, NC_SHUFFLE, 1, 1);
+}
+int main() { printf("Testing\n"); printf("%s\n",nc_strerror(0)); return 0; }
+EOF
+  TestProgram silent "  Checking NetCDF4/HDF5" "$CXX" "$CXXFLAGS ${LIB_INCL[$LNETCDF]}" testp.cpp "${LIB_FLAG[$LNETCDF]}"
+  if [ $? -eq 0 ] ; then
+    # HDF5 support is present.
+    # If we are going to build our own netcdf, it needs to know how to link
+    # HDF5.
+    if [ "${LIB_MAKE[$LNETCDF]}" = 'yes' ] ; then
+      LIB_FLAG[$LHDF5]="-lhdf5_hl -lhdf5 -ldl ${LIB_FLAG[$LZIP]}"
+    fi
+    return 0
+  fi
+  # If netcdf was specified, this means netcdf was built without hdf5 support
+  if [ "${LIB_STAT[$LNETCDF]}" = 'specified' ] ; then
+    ErrMsg "NetCDF in '${LIB_HOME[$LNETCDF]}' does not have HDF5 support."
+    ErrMsg "To use this NetCDF, disable HDF5 support with '-nohdf5'."
+    exit 1
+  fi
+  # Check for separate HDF5
+  cat > testp.cpp <<EOF
+#include <cstdio>
+#include "hdf5.h"
+void unused() {
+  hid_t file;
+  file = H5Fcreate("tmp", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+  herr_t status = H5Fclose(file);
+}
+int main() {
+  printf("Testing\n");
+  return 0;
+}
+EOF
+  TestProgram silent "  Checking HDF5" "$CXX" "$CXXFLAGS ${LIB_INCL[$LHDF5]}" testp.cpp "${LIB_FLAG[$LHDF5]}"
+  if [ $? -ne 0 ] ; then
+    if [ "${LIB_STAT[$LHDF5]}" = 'enabled' ] ; then
+      # See if there is already a version of HDF5 in CPPTRAJHOME
+      LIB_STAT[$LHDF5]='specified'
+      LIB_HOME[$LHDF5]=$CPPTRAJHOME
+      LIB_FLAG[$LHDF5]="$CPPTRAJHOME/lib/libhdf5_hl.a $CPPTRAJHOME/lib/${LIB_STTC[$LHDF5]} -ldl ${LIB_FLAG[$LZIP]}"
+      LIB_INCL[$LHDF5]="-I$CPPTRAJHOME/include"
+      TestProgram $BUILDTESTOPT "  Checking for bundled HDF5" "$CXX" "$CXXFLAGS ${LIB_INCL[$LHDF5]}" testp.cpp "${LIB_FLAG[$LHDF5]}"
+      if [ $? -ne 0 ] ; then
+        CheckRebuild "HDF5" "${LIB_FLAG[$LHDF5]}"
+        # See if user would like to get HDF5 
+        PREFIX=$CPPTRAJHOME CC=$CC CFLAGS=$CFLAGS LIBNAME='hdf5' \
+            SRCDIR=$HDF5_SRCDIR SRCTAR=$HDF5_SRCTAR URL=$HDF5_URL ./get_library.sh $REBUILDOPT $HDF5_OPTS
+        if [ $? -ne 0 ] ; then
+          ErrMsg "Building HDF5 failed."
+          exit 1
+        fi
+        # Test the built HDF5
+        TestProgram "  Checking built HDF5" "$CXX" "$CXXFLAGS ${LIB_INCL[$LHDF5]}" testp.cpp "${LIB_FLAG[$LHDF5]}"
+      fi
+    else
+      # HDF5 specified but failed.
+      TestProgErr "HDF5"
+    fi
+  fi
+}
+
 TestNetcdf() {
   cat > testp.cpp <<EOF
 #include <cstdio>
@@ -663,14 +758,35 @@ EOF
       LIB_HOME[$LNETCDF]=$CPPTRAJHOME
       LIB_FLAG[$LNETCDF]=$CPPTRAJHOME/lib/${LIB_STTC[$LNETCDF]}
       LIB_INCL[$LNETCDF]="-I$CPPTRAJHOME/include"
+      if [ "${LIB_STAT[$LHDF5]}" != 'off' ] ; then
+        # Netcdf needs to know where hdf5 is
+        LIB_FLAG[$LNETCDF]="${LIB_FLAG[$LNETCDF]} ${LIB_FLAG[$LHDF5]}"
+        LIB_INCL[$LNETCDF]="${LIB_INCL[$LNETCDF]} ${LIB_INCL[$LHDF5]}"
+      fi
       TestProgram $BUILDTESTOPT "  Checking for bundled NetCDF" "$CXX" "$CXXFLAGS ${LIB_INCL[$LNETCDF]}" testp.cpp "${LIB_FLAG[$LNETCDF]}"
       if [ $? -ne 0 ] ; then
         # Check if lib present but needs to be rebuilt
         CheckRebuild "NetCDF" "${LIB_FLAG[$LNETCDF]}"
-        # See if user would like to get netcdf
-        PREFIX=$CPPTRAJHOME CC=$CC CFLAGS=$CFLAGS LIBNAME='netcdf' \
-            SRCDIR=$NETCDF_SRCDIR SRCTAR=$NETCDF_SRCTAR URL=$NETCDF_URL ./get_library.sh $rebuild_netcdf $NETCDF_OPTS
-        if [ $? -ne 0 ] ; then
+        if [ "${LIB_STAT[$LHDF5]}" != 'off' ] ; then
+          # Get/compile netcdf4 with HDF5
+          # Some of the HDF5-related portions of NetCDF4 have race conditions,
+          # so override MAKE_COMMAND
+          if [ -z "$MAKE_COMMAND" ] ; then
+            echo "Setting MAKE_COMMAND to 'make -j1' to avoid NetCDF4/HDF5 race conditions."
+          else
+            echo "Overriding MAKE_COMMAND with 'make -j1' to avoid NetCDF4/HDF5 race conditions."
+          fi
+          PREFIX=$CPPTRAJHOME CC=$CC CFLAGS=$CFLAGS LIBNAME='netcdf' MAKE_COMMAND='make -j1' \
+              CPPFLAGS=${LIB_INCL[$LHDF5]} LDFLAGS="-L$CPPTRAJHOME/lib" \
+              SRCDIR=$NETCDF_SRCDIR SRCTAR=$NETCDF_SRCTAR URL=$NETCDF_URL ./get_library.sh $rebuild_netcdf $NETCDF4_OPTS
+          netcdferr=$?
+        else
+          # Get/compile netcdf3
+          PREFIX=$CPPTRAJHOME CC=$CC CFLAGS=$CFLAGS LIBNAME='netcdf' \
+              SRCDIR=$NETCDF_SRCDIR SRCTAR=$NETCDF_SRCTAR URL=$NETCDF_URL ./get_library.sh $rebuild_netcdf $NETCDF_OPTS
+          netcdferr=$?
+        fi
+        if [ $netcdferr -ne 0 ] ; then
           ErrMsg "Building NetCDF failed."
           exit 1
         fi
@@ -692,7 +808,16 @@ void unused() {int ncid; ncmpi_open(MPI_COMM_WORLD, "foo.nc", NC_NOWRITE, MPI_IN
 int main() { printf("Testing\n"); printf("%s\n",ncmpi_strerror(0)); return 0; }
 EOF
   TestProgram silent "  Checking Parallel NetCDF" "$CXX" "$CXXFLAGS ${LIB_INCL[$LPARANC]}" testp.cpp "${LIB_FLAG[$LPARANC]}"
-  if [ $? -ne 0 ] ; then
+  pnetcdfstat=$?
+  # Check if pnetcdf just needs netcdf support
+  if [ $pnetcdfstat -ne 0 -a "${LIB_STAT[$LPARANC]}" = 'specified' ] ; then
+    TestProgram silent "  Checking Parallel NetCDF (link to NetCDF)" "$CXX" "$CXXFLAGS ${LIB_INCL[$LPARANC]} ${LIB_INCL[$LNETCDF]}" testp.cpp "${LIB_FLAG[$LPARANC]} ${LIB_FLAG[$LNETCDF]}"
+    pnetcdfstat=$?
+    # If above check worked, pnetcdf should be linked OK with netcdf flags
+    return 0
+    #echo "DEBUG: Returned $?"
+  fi
+  if [ $pnetcdfstat -ne 0 ] ; then
     if [ "${LIB_STAT[$LPARANC]}" = 'optional' ] ; then
       WrnMsg "Parallel NetCDF test failed. CPPTRAJ will be built without parallel NetCDF."
       LIB_STAT[$LPARANC]='off'
@@ -1103,6 +1228,7 @@ TestLibraries() {
   fi
   if [ "${LIB_TEST[$LBZIP]}" = 'yes'     ] ; then TestBzlib ; fi
   if [ "${LIB_TEST[$LZIP]}" = 'yes'      ] ; then TestZlib ; fi
+  if [ "${LIB_TEST[$LHDF5]}" = 'yes'     ] ; then TestHdf5 ; fi
   if [ "${LIB_TEST[$LNETCDF]}" = 'yes'   ] ; then TestNetcdf ; fi
   if [ "${LIB_TEST[$LPARANC]}" = 'yes'   ] ; then TestPnetcdf ; fi
   if [ "${LIB_TEST[$LBLAS]}" = 'yes'     ] ; then TestMathlib ; fi
@@ -1425,6 +1551,12 @@ SetupLibraries() {
     echo "Warning: TNG depends on libz; disabling TNG."
     LIB_STAT[$LTNGFILE]='off'
   fi
+  if [ "${LIB_STAT[$LNETCDF]}" = 'off' -a "${LIB_STAT[$LHDF5]}" != 'off' ] ; then
+    # HDF5 needs netcdf
+    echo "Warning: HDF5 depends on NetCDF; disabling HDF5."
+    LIB_STAT[$LHDF5]='off'
+    LIB_TEST[$LHDF5]='no'
+  fi
 }
 
 #-------------------------------------------------------------------------------
@@ -1893,7 +2025,7 @@ SetupMKL() {
   LIB_FLAG[$LLAPACK]=''
   # If no FFTW, try using MKL
   if [ "$MKL_FFTW" != 'no' ] ; then
-    if [ "${LIB_STAT[$LFFTW3]}" = 'off' -o "${LIB_STAT[$LFFTW3]}" = 'optional' ] ; then
+    if [ "${LIB_STAT[$LFFTW3]}" = 'off' -o "${LIB_STAT[$LFFTW3]}" = 'optional' -o "${LIB_STAT[$LFFTW3]}" = 'enabled' ] ; then
       MKL_FFTW='yes'
       LIB_STAT[$LFFTW3]='enabled'
       LIB_FLAG[$LFFTW3]=''

devtools/ci/appveyor/build.bat

@@ -30,7 +30,7 @@ if %BUILD_TYPE% equ cmake-mingw (
 )
 
 if %BUILD_TYPE% equ configure-mingw (
-	sh -lc "./configure --with-netcdf=/mingw64/ --with-blas=/mingw64/ -openblas --with-bzlib=/mingw64/ --with-zlib=/mingw64 --with-arpack=/mingw64 --with-readline=/mingw64/ -nofftw3 -shared -windows gnu" || exit /b
+	sh -lc "./configure -nohdf5 --with-netcdf=/mingw64/ --with-blas=/mingw64/ -openblas --with-bzlib=/mingw64/ --with-zlib=/mingw64 --with-arpack=/mingw64 --with-readline=/mingw64/ -nofftw3 -shared -windows gnu" || exit /b
 	make libcpptraj -j2 || exit /b
 	make install -j2 || exit /b
 )

doc/cpptraj.bib

@@ -437,3 +437,10 @@ @article{Roe2022
 issue = {13},
 year = {2022}
 }
+
+@article{Roe2022a,
+author = {Roe, Daniel R. and Brooks, Bernard R.},
+journal = {Protein Science},
+title = {{ Quantifying the Effects of Lossy Compression on Energies Calculated from Molecular Dynamics Trajectories}},
+year = {2022}
+}