Skip to content

Commit

Permalink
hotfix: Temporarily skip datagrams generated by EC150 (#1357)
Browse files Browse the repository at this point in the history 
* change parse_ek to skip EC150 datagrams

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix pre-commit errors, update test

* fix bug to look for cconfiguration in self.config_datagram for ek60

* comment out print statement

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: ctuguinay <cmtuguinay2000@gmail.com>
  • Loading branch information
@leewujung @pre-commit-ci @ctuguinay
3 people authored Jul 15, 2024
1 parent f065cab commit d4cf8ea
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 31 deletions.
95 changes: 64 additions & 31 deletions echopype/convert/parse_base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,15 @@ def parse_raw(self):
self.config_datagram["timestamp"] = np.datetime64(
self.config_datagram["timestamp"].replace(tzinfo=None), "[ns]"
)

# Only EK80 files have configuration in self.config_datagram
if "configuration" in self.config_datagram:
# Remove EC150 (ADCP) from config
channel_id = list(self.config_datagram["configuration"].keys())
channel_id_rm = [ch for ch in channel_id if "EC150" in ch]
for ch in channel_id_rm:
_ = self.config_datagram["configuration"].pop(ch)

for v in self.config_datagram["configuration"].values():
if "pulse_duration" not in v and "pulse_length" in v:
# it seems like sometimes this field can appear with the name "pulse_length"
Expand Down Expand Up @@ -476,6 +484,12 @@ def _read_datagrams(self, fid):
new_datagram["timestamp"].replace(tzinfo=None), "[ns]"
)

# # For debugging EC150 datagrams
# if new_datagram["type"].startswith("XML") and "subtype" in new_datagram:
# print(f"{new_datagram['type']} - {new_datagram['subtype']}")
# else:
# print(new_datagram["type"])

num_datagrams_parsed += 1

# XML datagrams store environment or instrument parameters for EK80
Expand All @@ -485,7 +499,14 @@ def _read_datagrams(self, fid):
self.environment["xml"] = new_datagram["xml"]
self.environment["timestamp"] = new_datagram["timestamp"]
elif new_datagram["subtype"] == "parameter":
current_parameters = new_datagram["parameter"]
if "EC150" not in new_datagram["parameter"]["channel_id"]:
# print(
# f"{new_datagram['parameter']['channel_id']} from XML-parameter "
# "-- NOT SKIPPING"
# )
current_parameters = new_datagram["parameter"]
# else:
# print(f"{new_datagram['parameter']['channel_id']} from XML-parameter")

# RAW0 datagrams store raw acoustic data for a channel for EK60
elif new_datagram["type"].startswith("RAW0"):
Expand All @@ -502,33 +523,41 @@ def _read_datagrams(self, fid):
# - RAW3
# RAW3 datagrams store raw acoustic data for a channel for EK80
elif new_datagram["type"].startswith("RAW3"):
curr_ch_id = new_datagram["channel_id"]
# Check if the proceeding Parameter XML does not
# match with data in this RAW3 datagram
if current_parameters["channel_id"] != curr_ch_id:
raise ValueError("Parameter ID does not match RAW")

# Save channel-specific ping time
self.ping_time[curr_ch_id].append(new_datagram["timestamp"])

# Append ping by ping data
new_datagram.update(current_parameters)
self._append_channel_ping_data(new_datagram)
if "EC150" not in new_datagram["channel_id"]:
# print(f"{new_datagram['channel_id']} from RAW3 -- NOT SKIPPING")
curr_ch_id = new_datagram["channel_id"]
# Check if the proceeding Parameter XML does not
# match with data in this RAW3 datagram
if current_parameters["channel_id"] != curr_ch_id:
raise ValueError("Parameter ID does not match RAW")

# Save channel-specific ping time
self.ping_time[curr_ch_id].append(new_datagram["timestamp"])

# Append ping by ping data
new_datagram.update(current_parameters)
self._append_channel_ping_data(new_datagram)
# else:
# print(f"{new_datagram['channel_id']} from RAW3")

# RAW4 datagrams store raw transmit pulse for a channel for EK80
elif new_datagram["type"].startswith("RAW4"):
curr_ch_id = new_datagram["channel_id"]
# Check if the proceeding Parameter XML does not
# match with data in this RAW4 datagram
if current_parameters["channel_id"] != curr_ch_id:
raise ValueError("Parameter ID does not match RAW")

# Ping time is identical to the immediately following RAW3 datagram
# so does not need to be stored separately

# Append ping by ping data
new_datagram.update(current_parameters)
self._append_channel_ping_data(new_datagram, raw_type="transmit")
if "EC150" not in new_datagram["channel_id"]:
# print(f"{new_datagram['channel_id']} from RAW4 -- NOT SKIPPING")
curr_ch_id = new_datagram["channel_id"]
# Check if the proceeding Parameter XML does not
# match with data in this RAW4 datagram
if current_parameters["channel_id"] != curr_ch_id:
raise ValueError("Parameter ID does not match RAW")

# Ping time is identical to the immediately following RAW3 datagram
# so does not need to be stored separately

# Append ping by ping data
new_datagram.update(current_parameters)
self._append_channel_ping_data(new_datagram, raw_type="transmit")
# else:
# print(f"{new_datagram['channel_id']} from RAW4")

# NME datagrams store ancillary data as NMEA-0817 style ASCII data.
elif new_datagram["type"].startswith("NME"):
Expand All @@ -545,12 +574,16 @@ def _read_datagrams(self, fid):

# FIL datagrams contain filters for processing bascatter data for EK80
elif new_datagram["type"].startswith("FIL"):
self.fil_coeffs[new_datagram["channel_id"]][new_datagram["stage"]] = new_datagram[
"coefficients"
]
self.fil_df[new_datagram["channel_id"]][new_datagram["stage"]] = new_datagram[
"decimation_factor"
]
if "EC150" not in new_datagram["channel_id"]:
# print(f"{new_datagram['channel_id']} from FIL -- NOT SKIPPING")
self.fil_coeffs[new_datagram["channel_id"]][new_datagram["stage"]] = (
new_datagram["coefficients"]
)
self.fil_df[new_datagram["channel_id"]][new_datagram["stage"]] = new_datagram[
"decimation_factor"
]
# else:
# print(f"{new_datagram['channel_id']} from FIL")

# TAG datagrams contain time-stamped annotations inserted via the recording software
elif new_datagram["type"].startswith("TAG"):
Expand Down
13 changes: 13 additions & 0 deletions echopype/tests/convert/test_convert_ek80.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -442,3 +442,16 @@ def test_convert_ek80_mru1(ek80_path):
np.all(echodata["Platform"]["roll"].data == np.array(parser.mru["roll"]))
np.all(echodata["Platform"]["vertical_offset"].data == np.array(parser.mru["heave"]))
np.all(echodata["Platform"]["heading"].data == np.array(parser.mru["heading"]))


def test_skip_ec150(ek80_path):
"""Make sure we skip EC150 datagrams correctly."""
ek80_mru1_path = str(ek80_path.joinpath("RL2407_ADCP-D20240709-T150437.raw"))
echodata = open_raw(raw_file=ek80_mru1_path, sonar_model='EK80')

assert "EC150" not in echodata["Sonar/Beam_group1"]["channel"].values
assert "backscatter_i" in echodata["Sonar/Beam_group1"].data_vars
assert (
echodata["Sonar/Beam_group1"].dims
== {'channel': 1, 'ping_time': 2, 'range_sample': 115352, 'beam': 4}
)

0 comments on commit d4cf8ea

Please sign in to comment.