Merge remote-tracking branch 'origin/develop' into feature/grib_time_…

…processing_generalisation
ecmwf · Sep 25, 2024 · ab234d0 · ab234d0
2 parents c11b2e6 + a0023ff
commit ab234d0
Show file tree

Hide file tree

Showing 180 changed files with 2,751 additions and 1,922 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -69,6 +69,8 @@ jobs:
     secrets:
       url_debian_11: ${{ secrets.NEXUS_TEST_REPO_NIGHTLY_URL_DEBIAN_11 }}
       token_debian_11: ${{ secrets.NEXUS_TEST_REPO_UPLOAD_TOKEN }}
+      url_debian_12: ${{ secrets.NEXUS_TEST_REPO_URL_DEBIAN_12 }}
+      token_debian_12: ${{ secrets.NEXUS_TEST_REPO_UPLOAD_TOKEN }}
       url_centos_7: ${{ secrets.NEXUS_TEST_REPO_NIGHTLY_URL_CENTOS_7 }}
       token_centos_7: ${{ secrets.NEXUS_TEST_REPO_UPLOAD_TOKEN }}
       url_rocky_8: ${{ secrets.NEXUS_TEST_REPO_NIGHTLY_URL_ROCKY_8 }}

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -18,7 +18,16 @@
 
 cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
 
-find_package( ecbuild 3.7 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild)
+find_package( ecbuild 3.7 HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild)
+if(NOT ecbuild_FOUND)
+    message(STATUS "Fetching ecbuild...")
+    include(FetchContent)
+    FetchContent_Populate(ecbuild
+        GIT_REPOSITORY https://github.com/ecmwf/ecbuild.git
+        GIT_TAG        3.8.5
+    )
+    find_package( ecbuild 3.7 REQUIRED HINTS ${ecbuild_SOURCE_DIR})
+endif()
 
 # Initialise project
 project( eccodes LANGUAGES CXX )
@@ -509,11 +518,8 @@ ecbuild_info("   |  PRE-RELEASE CANDIDATE   |")
 ecbuild_info("   +--------------------------+")
 ecbuild_info("")
 
-ecbuild_info("   +--------------------------------------+")
-ecbuild_info("   | Please note:                         |")
-ecbuild_info("   | For Python3 support, first install   |")
-ecbuild_info("   | ecCodes and then install the Python  |")
-ecbuild_info("   | bindings from PyPI with:             |")
-ecbuild_info("   |   $ pip3 install eccodes             |")
-ecbuild_info("   +--------------------------------------+")
+ecbuild_info("Please note:")
+ecbuild_info(" For Python3 support, you must install the Python bindings.")
+ecbuild_info(" See:")
+ecbuild_info("  https://confluence.ecmwf.int/display/ECC/ecCodes+installation")
 ecbuild_info("")
diff --git a/definitions/bufr/rdb_key.def b/definitions/bufr/rdb_key.def
@@ -1,4 +1,5 @@
 # RDB stands for REPORT DATA BASE
+# See https://confluence.ecmwf.int/pages/viewpage.action?pageId=24316441
 unsigned[1]   rdbType : dump; 
 unsigned[1]   oldSubtype: dump;
 

diff --git a/definitions/bufr/section.3.def b/definitions/bufr/section.3.def
@@ -50,7 +50,7 @@ unsigned[1]     reservedSection3 = 0;
 unsigned[2]     numberOfSubsets : dump;
 alias ls.numberOfSubsets=numberOfSubsets;
 
-if (section2Present && bufrHeaderCentre==98 && section2Length==52) {
+if (section2Present && bufrHeaderCentre == 98 && section2Length == 52) {
     if ( rdbType == 2 || rdbType == 3 || rdbType == 8 || rdbType == 12 || rdbType == 30 ) {
         transient isSatelliteType=1;
     } else {
@@ -69,10 +69,11 @@ if (section2Present && bufrHeaderCentre==98 && section2Length==52) {
         meta localLatitude2 bits(keyMore,32,25,-9000000,100000) : dump,no_copy;
 
         # This rule is taken from BUFRDC. See bufrdc_wmo/buukey.F
-        # Also see ECC-686
-        if (oldSubtype == 255 || numberOfSubsets>255 ||
-            ( oldSubtype>=121 && oldSubtype <=130 )  ||
-            oldSubtype==31) {
+        # See ECC-686
+        # and https://confluence.ecmwf.int/pages/viewpage.action?pageId=24316441
+        if (oldSubtype == 255 || numberOfSubsets > 255 ||
+            ( oldSubtype >= 121 && oldSubtype <= 130 ) ||
+            oldSubtype == 31) {
             meta ls.localNumberOfObservations bits(keySat,0,16) : dump,long_type,no_copy;
             meta ls.satelliteID bits(keySat,16,16) : dump,long_type,no_copy;
         } else {
@@ -114,16 +115,16 @@ meta bufrdcExpandedDescriptors bufrdc_expanded_descriptors(expandedCodes) : no_c
 #smart_table NAME (VALUES,FILE_NAME,MASTER_DIRECTORY,LOCAL_DIRECTORY,WIDTH_OF_CODE_IN_BITS,EXTRA_DIRECTORY,EXTRA_FILE_NAME);
 #Note: The WIDTH_OF_CODE_IN_BITS has to be big enough so 2^width > the highest BUFR descriptor code
 smart_table expandedOriginalCodes (expandedCodes,"element.table",tablesMasterDir,tablesLocalDir,18,rootTablesDir,"operators.table") :no_copy;
-meta expandedAbbreviations         smart_table_column(expandedOriginalCodes,0,1) : string_type,no_copy;
-meta expandedTypes                 smart_table_column(expandedOriginalCodes,1,0) : string_type,no_copy;
-meta expandedNames                 smart_table_column(expandedOriginalCodes,2,0) : string_type,no_copy;
-meta expandedUnits                 smart_table_column(expandedOriginalCodes,3,0) : string_type,no_copy;
-meta expandedOriginalScales        smart_table_column(expandedOriginalCodes,4,0) : long_type,no_copy;
-meta expandedOriginalReferences    smart_table_column(expandedOriginalCodes,5,0) : long_type,no_copy;
-meta expandedOriginalWidths        smart_table_column(expandedOriginalCodes,6,0) : long_type,no_copy;
-meta expandedCrex_units            smart_table_column(expandedOriginalCodes,7,0) : long_type,no_copy;
-meta expandedCrex_scales           smart_table_column(expandedOriginalCodes,8,0) : long_type,no_copy;
-meta expandedCrex_widths           smart_table_column(expandedOriginalCodes,9,0) : long_type,no_copy;
+meta expandedAbbreviations        smart_table_column(expandedOriginalCodes,0,1) : string_type,no_copy;
+meta expandedTypes                smart_table_column(expandedOriginalCodes,1,0) : string_type,no_copy;
+meta expandedNames                smart_table_column(expandedOriginalCodes,2,0) : string_type,no_copy;
+meta expandedUnits                smart_table_column(expandedOriginalCodes,3,0) : string_type,no_copy;
+meta expandedOriginalScales       smart_table_column(expandedOriginalCodes,4,0) : long_type,no_copy;
+meta expandedOriginalReferences   smart_table_column(expandedOriginalCodes,5,0) : long_type,no_copy;
+meta expandedOriginalWidths       smart_table_column(expandedOriginalCodes,6,0) : long_type,no_copy;
+meta expandedCrex_units           smart_table_column(expandedOriginalCodes,7,0) : long_type,no_copy;
+meta expandedCrex_scales          smart_table_column(expandedOriginalCodes,8,0) : long_type,no_copy;
+meta expandedCrex_widths          smart_table_column(expandedOriginalCodes,9,0) : long_type,no_copy;
 
 position endDescriptors;
 section_padding section3Padding;

diff --git a/definitions/grib1/boot.def b/definitions/grib1/boot.def
@@ -16,7 +16,7 @@ constant three=1 : hidden;
 constant eight=8 : hidden;
 constant eleven=11 : hidden;
 constant epsPoint=1 : hidden;
-constant epsContinous=11 : hidden;
+constant epsContinuous=11 : hidden;
 constant epsStatisticsPoint=2 : hidden;
 constant epsStatisticsContinous=12 : hidden;
 

diff --git a/definitions/grib1/data.grid_simple_matrix.def b/definitions/grib1/data.grid_simple_matrix.def
@@ -114,7 +114,6 @@ if(matrixOfValues == 0)
 
 # From GRIBEX:
 #  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-#  !                                                !
 #  ! This is the WMO definition, but it is entirely !
 #  ! inadequate when secondary bit maps are present !
 #  ! eg 3x3 global grid with a matrix of values     !
@@ -129,7 +128,6 @@ if(matrixOfValues == 0)
 #  !      missing').                                !
 #  !     This definition will accommodate a 1x1     !
 #  !     degree global grid.                        !
-#  !                                                !
 #  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 #
 	  constant datumSize             = NC*NR;

diff --git a/definitions/grib1/gds_not_present_bitmap.def b/definitions/grib1/gds_not_present_bitmap.def
@@ -10,10 +10,11 @@ transient numberOfUnusedBitsAtEndOfSection3 = 0 : read_only;
 transient tableReference = 0;
 
 #position offsetBeforeBitmap;
-meta bitmap gds_not_present_bitmap( missingValue,numberOfValues,
-                                    numberOfPoints,
-                                    latitudeOfFirstGridPoint,
-                                    Ni,numberOfUnusedBitsAtEndOfSection3) : read_only;
+meta bitmap gds_not_present_bitmap(
+    missingValue,numberOfValues,
+    numberOfPoints,
+    latitudeOfFirstGridPoint,
+    Ni,numberOfUnusedBitsAtEndOfSection3) : read_only;
 
 #position offsetAfterBitmap;
 #padtoeven padding_sec3_1(offsetSection3,section3Length);

diff --git a/definitions/grib1/grid_definition_192.78.def b/definitions/grib1/grid_definition_192.78.def
@@ -40,6 +40,6 @@ transient numberOfPoints= nd *(Ni + 1) * (Ni + 1);
 alias numberOfDataPoints=numberOfPoints;
 
 meta numberOfValues
-number_of_values(values,bitsPerValue,numberOfDataPoints,
-                 bitmapPresent,bitmap,numberOfCodedValues) : dump;
-
+number_of_values(
+    values,bitsPerValue,numberOfDataPoints,
+    bitmapPresent,bitmap,numberOfCodedValues) : dump;
diff --git a/definitions/grib1/grid_definition_193.98.def b/definitions/grib1/grid_definition_193.98.def
@@ -79,11 +79,11 @@ if(missing(Ni)){
    nearest latlon_reduced(values,radius,Nj,pl);
 } else {
    transient iteratorDisableUnrotate = 0 : hidden; # ECC-808
-   iterator latlon(numberOfPoints,missingValue,values,longitudeFirstInDegrees,iInc ,
-                   Ni,Nj,iScansNegatively ,
-                   latitudeFirstInDegrees,DjInDegrees,jScansPositively,jPointsAreConsecutive,
-                   isRotatedGrid, angleOfRotation,
-                   latitudeOfSouthernPoleInDegrees,longitudeOfSouthernPoleInDegrees);
+   iterator latlon(numberOfPoints,missingValue,values,longitudeFirstInDegrees,iInc,
+         Ni,Nj,iScansNegatively,
+         latitudeFirstInDegrees,DjInDegrees,jScansPositively,jPointsAreConsecutive,
+         isRotatedGrid, angleOfRotation,
+         latitudeOfSouthernPoleInDegrees,longitudeOfSouthernPoleInDegrees);
    nearest regular(values,radius,Ni,Nj);
 }
 meta latLonValues latlonvalues(values);

diff --git a/definitions/grib1/local.214.1.def b/definitions/grib1/local.214.1.def
@@ -32,7 +32,7 @@ if (stepType is "instant" ) {
   }
 } else {
   if (numberOfForecastsInEnsemble!=0) {
-    alias productDefinitionTemplateNumber=epsContinous;
+    alias productDefinitionTemplateNumber=epsContinuous;
   }
 }
 

diff --git a/definitions/grib1/local.34.1.def b/definitions/grib1/local.34.1.def
@@ -40,7 +40,7 @@ if (stepType is "instant" ) {
 			} else {
 				alias typeOfEnsembleForecast=three;
 			}
-			alias productDefinitionTemplateNumber=epsContinous;
+			alias productDefinitionTemplateNumber=epsContinuous;
 		} else {
 			alias productDefinitionTemplateNumber=eight;
 		}

diff --git a/definitions/grib1/local.98.1.def b/definitions/grib1/local.98.1.def
@@ -41,7 +41,7 @@ if (stepType is "instant" ) {
             } else {
                 alias typeOfEnsembleForecast=three;
             }
-            alias productDefinitionTemplateNumber=epsContinous;
+            alias productDefinitionTemplateNumber=epsContinuous;
         } else {
             alias productDefinitionTemplateNumber=eight;
         }

diff --git a/definitions/grib1/local.98.18.def b/definitions/grib1/local.98.18.def
@@ -10,7 +10,7 @@ constant grib2LocalSectionNumber=18;
 if (stepType is "instant" ) {
   alias productDefinitionTemplateNumber=epsPoint;
 } else {
-  alias productDefinitionTemplateNumber=epsContinous;
+  alias productDefinitionTemplateNumber=epsContinuous;
 }
 template mars_labeling "grib1/mars_labeling.def";
 

diff --git a/definitions/grib1/local.98.218.def b/definitions/grib1/local.98.218.def
@@ -10,7 +10,7 @@ constant grib2LocalSectionNumber=18;
 if (stepType is "instant" ) {
   alias productDefinitionTemplateNumber=epsPoint;
 } else {
-   alias productDefinitionTemplateNumber=epsContinous;
+   alias productDefinitionTemplateNumber=epsContinuous;
 }
 template mars_labeling "grib1/mars_labeling.def";
 

diff --git a/definitions/grib1/local.98.26.def b/definitions/grib1/local.98.26.def
@@ -14,7 +14,7 @@ constant grib2LocalSectionNumber=26;
 if (stepType is "instant" ) {
   alias productDefinitionTemplateNumber=epsPoint;
 } else {
-  alias productDefinitionTemplateNumber=epsContinous;
+  alias productDefinitionTemplateNumber=epsContinuous;
 }
 
 constant wrongPadding=1 : hidden;

diff --git a/definitions/grib1/local.98.30.def b/definitions/grib1/local.98.30.def
@@ -23,7 +23,7 @@ if (stepType is "instant" ) {
   if (type is "em" || type is "es" ) {
     alias productDefinitionTemplateNumber=epsStatisticsContinous;
   } else {
-    alias productDefinitionTemplateNumber=epsContinous;
+    alias productDefinitionTemplateNumber=epsContinuous;
   }
 }
 

diff --git a/definitions/grib1/local.98.36.def b/definitions/grib1/local.98.36.def
@@ -47,7 +47,7 @@ if (stepType is "instant" ) {
             } else {
                 alias typeOfEnsembleForecast=three;
             }
-            alias productDefinitionTemplateNumber=epsContinous;
+            alias productDefinitionTemplateNumber=epsContinuous;
         } else {
             alias productDefinitionTemplateNumber=eight;
         }

diff --git a/definitions/grib1/local/ecmf/3.table b/definitions/grib1/local/ecmf/3.table
@@ -42,10 +42,12 @@
 141 141 Layer between two isobaric surfaces (mixed precision) pressure of top, in kPa 1100hPa minus pressure of bottom, in hPa
 # 142-159 Reserved
 160 dp Depth below sea level meters (2 octets)
-# 161-199Reserved
+# 161-199 Reserved
 200 sfc Entire atmosphere considered as a single layer 0 (2 octets)
 201 201 Entire ocean considered as a single layer 0 (2 octets)
-# 202-209 Reserved
+202 al Abstract Single Level
+203 al Abstract Multiple Level
+# 204-209 Reserved
 210 pl Isobaric surface (Pa) (ECMWF extension)
 # 211-254 Reserved for local use
 211 wv Ocean wave level (ECMWF extension)

diff --git a/definitions/grib1/local_no_mars.98.1.def b/definitions/grib1/local_no_mars.98.1.def
@@ -48,7 +48,7 @@ if (stepType is "instant" ) {
 			} else {
 				alias typeOfEnsembleForecast=three;
 			}
-			alias productDefinitionTemplateNumber=epsContinous;
+			alias productDefinitionTemplateNumber=epsContinuous;
 		} else {
 			alias productDefinitionTemplateNumber=eight;
 		}

diff --git a/definitions/grib1/predefined_grid.def b/definitions/grib1/predefined_grid.def
@@ -24,9 +24,9 @@ constant dataRepresentationType  = 0;
 # (according to data representation type - octet 6 above)
 
 # grib 1 -> 2
-constant gridDefinitionTemplateNumber     = 0;
+constant gridDefinitionTemplateNumber = 0;
 
-# START 1/grid_definition.latitude_longitude_grid ----------------------------------------------------------------------
+# START 1/grid_definition.latitude_longitude_grid
 # GRID DEFINITION latitude/longitude grid (or equidistant cylindrical)
 
 alias numberOfPointsAlongAParallel=Ni;
@@ -122,6 +122,6 @@ constant tableReference = 0;
 
 #position offsetBeforeBitmap;
 # meta bitmap gds_not_present_bitmap( missingValue,numberOfValues,
-#                                     numberOfPoints,
-#                                     latitudeOfFirstGridPoint,
-#                                     Ni,numberOfUnusedBitsAtEndOfSection3) : read_only;
+# numberOfPoints,
+# latitudeOfFirstGridPoint,
+# Ni,numberOfUnusedBitsAtEndOfSection3) : read_only;
diff --git a/definitions/grib1/section.1.def b/definitions/grib1/section.1.def
@@ -105,7 +105,8 @@ if( indicatorOfTypeOfLevel == 109 ||
     indicatorOfTypeOfLevel == 100 ||
     indicatorOfTypeOfLevel == 110 ||
     indicatorOfTypeOfLevel == 113 ||
-    indicatorOfTypeOfLevel == 117)
+    indicatorOfTypeOfLevel == 117 ||
+    indicatorOfTypeOfLevel == 203)
 {
     alias mars.levelist = level;
 }

diff --git a/definitions/grib1/typeOfLevel.def b/definitions/grib1/typeOfLevel.def
@@ -34,3 +34,5 @@
 'depthBelowSea' = {indicatorOfTypeOfLevel=160;}
 'entireAtmosphere' = {indicatorOfTypeOfLevel=200;}
 'entireOcean' = {indicatorOfTypeOfLevel=201;}
+'abstractSingleLevel' = {indicatorOfTypeOfLevel=202; level=missing();}
+'abstractMultipleLevels' = {indicatorOfTypeOfLevel=203;}