Fix time issue, update test data sets

pyglider/seaexplorer.py

@@ -337,7 +337,7 @@ def raw_to_timeseries(indir, outdir, deploymentyaml, kind='raw',
         raise ValueError(f"timebase source: {ncvar['timebase']['source']} not found in pld1 columns")
     vals = sensor.select([ncvar['timebase']['source']]).to_numpy()[:, 0]
     indctd = np.where(~np.isnan(vals))[0]
-    ds['time'] = (('time'), sensor.select('time').to_numpy()[indctd, 0], attr)
+    ds['time'] = (('time'), sensor.select('time').to_numpy()[indctd, 0].astype('datetime64[ns]'), attr)
     thenames = list(ncvar.keys())
     # Check yaml to see if interpolate has been set to True
     if "interpolate" in thenames:
@@ -368,7 +368,7 @@ def raw_to_timeseries(indir, outdir, deploymentyaml, kind='raw',
                     sensor_sub_grouped = sensor_sub.with_columns(
                         pl.col('time').to_physical()
                     ).group_by(
-                        by=pl.col('coarse_ints'),
+                        pl.col('coarse_ints'),
                         maintain_order=True
                     ).mean().with_columns(
                         pl.col('time').cast(pl.Datetime('ms'))
@@ -492,8 +492,7 @@ def raw_to_timeseries(indir, outdir, deploymentyaml, kind='raw',
         if 'units' in ds.ad2cp_time.attrs.keys():
             ds.ad2cp_time.attrs.pop('units')
     ds.to_netcdf(outname, 'w',
-                 encoding={'time': {'units':
-                                        'seconds since 1970-01-01T00:00:00Z'}})
+                 encoding={'time': {'units': 'milliseconds since 1970-01-01T00:00:00Z'}})
     return outname
 
 

pyglider/slocum.py

@@ -760,7 +760,7 @@ def raw_to_timeseries(indir, outdir, deploymentyaml, *,
     ds = ds.assign_coords(depth=ds.depth)
 
     ds['time'] = (ds.time.values.astype('timedelta64[s]') +
-                  np.datetime64('1970-01-01T00:00:00'))
+                  np.datetime64('1970-01-01T00:00:00')).astype('datetime64[ns]')
     ds = utils.fill_metadata(ds, deployment['metadata'], device_data)
     start = ds['time'].values[0]
     end = ds['time'].values[-1]
@@ -887,8 +887,8 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *,
 
             # interpolate only over those gaps that are smaller than 'maxgap'
             _t, _ = dbd.get(ncvar[name]['source'])
-            tg_ind = utils.find_gaps(_t,time,maxgap) 
-            val[tg_ind] = np.nan            
+            tg_ind = utils.find_gaps(_t,time,maxgap)
+            val[tg_ind] = np.nan
 
             val = utils._zero_screen(val)
             val = convert(val)
@@ -919,8 +919,8 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *,
 
     # screen out-of-range times; these won't convert:
     ds['time'] = ds.time.where((ds.time>0) & (ds.time<6.4e9), np.NaN)
-    ds['time'] = (('time'), ds.time.values.astype('timedelta64[s]') +
-                  np.datetime64('1970-01-01T00:00:00'), attr)
+    ds['time'] = (('time'), (ds.time.values.astype('timedelta64[s]') +
+                  np.datetime64('1970-01-01T00:00:00')).astype('datetime64[ns]'), attr)
 
     ds = utils.fill_metadata(ds, deployment['metadata'], device_data)
     start = ds['time'].values[0]

tests/environment.yml

@@ -17,4 +17,4 @@ dependencies:
   - matplotlib
   - polars
   - pip:
-    - dbdreader<0.4.15
+    - dbdreader

tests/example-data/example-slocum-littleendian/deployment.yml

@@ -101,7 +101,6 @@ netcdf_variables:
     units:        degrees_north
     axis:         Y
     coordinates:   time depth latitude longitude
-    conversion:   nmea2deg
     comment:     "Estimated between surface fixes"
     observation_type: measured
     platform:     platform
@@ -117,7 +116,6 @@ netcdf_variables:
     units:        degrees_east
     axis:         X
     coordinates:  time depth latitude longitude
-    conversion:   nmea2deg
     comment:     "Estimated between surface fixes"
     observation_type: measured
     platform:     platform

tests/example-data/example-slocum/deploymentRealtime.yml

@@ -97,7 +97,6 @@ netcdf_variables:
     units:        degrees_north
     axis:         Y
     coordinates:   time depth latitude longitude
-    conversion:   nmea2deg
     comment:     "Estimated between surface fixes"
     observation_type: measured
     platform:     platform
@@ -113,7 +112,6 @@ netcdf_variables:
     units:        degrees_east
     axis:         X
     coordinates:  time depth latitude longitude
-    conversion:   nmea2deg
     comment:     "Estimated between surface fixes"
     observation_type: measured
     platform:     platform