From 77c56e7fd39b7dbc1da51174b953f2957d4459e8 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Tue, 24 May 2022 19:56:10 -0400 Subject: [PATCH] Fix HDF5 write WINDOWS when running in parallel --- src/TimeSeries.C | 14 +++++++++++--- src/moptmain.C | 14 ++++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/TimeSeries.C b/src/TimeSeries.C index df444d0b..3488cc4c 100644 --- a/src/TimeSeries.C +++ b/src/TimeSeries.C @@ -512,9 +512,17 @@ void TimeSeries::writeWindows( string suffix ) // int ret = createWriteAttr( grp, "WINDOWS", H5T_NATIVE_DOUBLE, data_space, windows ); // if( ret < 0 ) { - int ret = openWriteAttr(grp, "WINDOWS", H5T_NATIVE_DOUBLE, windows); - if( ret < 0 ) - std::cout << "TimeSeries::writeWindows, Error could not create/open data space" << std::endl; + if( H5Lexists(grp, "WINDOWS", H5P_DEFAULT) ) { + int ret = openWriteAttr(grp, "WINDOWS", H5T_NATIVE_DOUBLE, windows); + if( ret < 0 ) + std::cout << "TimeSeries::writeWindows, Error could not create/open data space" << std::endl; + } + else { + hsize_t nelements=4; + hid_t data_space = H5Screate_simple(1, &nelements, NULL); + int ret = createWriteAttr( grp, "WINDOWS", H5T_NATIVE_DOUBLE, data_space, windows ); + H5Sclose(data_space); + } } H5Gclose(grp); } diff --git a/src/moptmain.C b/src/moptmain.C index 1aecd5e7..1116410f 100644 --- a/src/moptmain.C +++ b/src/moptmain.C @@ -2192,9 +2192,12 @@ int main(int argc, char **argv) nlcg( simulation, nspar, nmpars, xs, nmpard, xm, GlobalSources, GlobalTimeSeries, GlobalObservations, myRank, mopt ); - for( int e=0 ; e < GlobalTimeSeries.size(); e++ ) - for( int m=0 ; m < GlobalTimeSeries[e].size() ; m++ ) - GlobalTimeSeries[e][m]->writeWindows(); + for( int e=0 ; e < GlobalTimeSeries.size(); e++ ) + for( int m=0 ; m < GlobalTimeSeries[e].size() ; m++ ) { + // This barrier is needed as each process needs to open and write to the HDF5 file independently, and one at a time + MPI_Barrier(simulation.m_1d_communicator); + GlobalTimeSeries[e][m]->writeWindows(); + } sw4_profile->time_stamp("Done optimizer"); sw4_profile->flush(); @@ -2318,8 +2321,11 @@ int main(int argc, char **argv) simulation.solveTT(GlobalSources[e][0], GlobalObservations[e], coarse, nmpars, mopt->m_mp, mopt->get_wave_mode(), e, simulation.getRank()); - for( int s=0 ; s < GlobalObservations[e].size() ; s++) + for( int s=0 ; s < GlobalObservations[e].size() ; s++) { + // This barrier is needed as each process needs to open and write to the HDF5 file independently, and one at a time + MPI_Barrier(simulation.m_1d_communicator); GlobalObservations[e][s]->writeWindows(); + } } } }