From 2b077a85a8f6604a4dec8bdae1056b257be43653 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Thu, 19 Feb 2026 10:13:29 -0700 Subject: [PATCH 1/3] Hotfix for ancillary files - wasn't getting the right file, was falling back to defaults incorrectly --- imap_processing/cli.py | 2 +- imap_processing/glows/l1b/glows_l1b_data.py | 46 ++++++++++++++++++++- imap_processing/glows/l2/glows_l2.py | 1 + 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/imap_processing/cli.py b/imap_processing/cli.py index f44dce139..cece0fd92 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -739,7 +739,7 @@ def do_processing( datasets = [glows_l1b_de(input_dataset, conversion_table_dict)] if self.data_level == "l2": - science_files = dependencies.get_file_paths(source="glows") + science_files = dependencies.get_file_paths(source="glows", data_type="l1b") if len(science_files) != 1: raise ValueError( f"GLOWS L2 requires exactly one input science file, " diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 429173888..50f9cee2e 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -99,22 +99,66 @@ def __post_init__(self, pipeline_dataset: xr.Dataset) -> None: Dataset containing pipeline settings data variables. """ # Extract active bad-angle flags (default to all True if not present) + _angle_flag_names = [ + "is_close_to_uv_source", + "is_inside_excluded_region", + "is_excluded_by_instr_team", + "is_suspected_transient", + ] if "active_bad_angle_flags" in pipeline_dataset.data_vars: self.active_bad_angle_flags = list( pipeline_dataset["active_bad_angle_flags"].values ) + elif any( + f"active_bad_angle_flags_{n}" in pipeline_dataset.data_vars + for n in _angle_flag_names + ): + # Flattened format from convert_json_to_dataset + self.active_bad_angle_flags = [ + bool(pipeline_dataset[f"active_bad_angle_flags_{name}"].values) + for name in _angle_flag_names + ] else: # Default: all 4 bad-angle flags are active self.active_bad_angle_flags = [True, True, True, True] # Extract active bad-time flags (default to all True if not present) + _time_flag_names = [ + "is_pps_missing", + "is_time_status_missing", + "is_phase_missing", + "is_spin_period_missing", + "is_overexposed", + "is_direct_event_non_monotonic", + "is_night", + "is_hv_test_in_progress", + "is_test_pulse_in_progress", + "is_memory_error_detected", + "is_generated_on_ground", + "is_beyond_daily_statistical_error", + "is_temperature_std_dev_beyond_threshold", + "is_hv_voltage_std_dev_beyond_threshold", + "is_spin_period_std_dev_beyond_threshold", + "is_pulse_length_std_dev_beyond_threshold", + "is_spin_period_difference_beyond_threshold", + ] if "active_bad_time_flags" in pipeline_dataset.data_vars: self.active_bad_time_flags = list( pipeline_dataset["active_bad_time_flags"].values ) + elif any( + f"active_bad_time_flags_{n}" in pipeline_dataset.data_vars + for n in _time_flag_names + ): + # Flattened format from convert_json_to_dataset + self.active_bad_time_flags = [ + bool(pipeline_dataset[f"active_bad_time_flags_{name}"].values) + for name in _time_flag_names + ] + print(self.active_bad_time_flags) else: # Default: assume all bad-time flags are active - self.active_bad_time_flags = [True] * 16 # Typical number of bad-time flags + self.active_bad_time_flags = [True] * FLAG_LENGTH # Extract sunrise/sunset offsets (default to 0.0 if not present) self.sunrise_offset = float(pipeline_dataset.get("sunrise_offset", 0.0)) diff --git a/imap_processing/glows/l2/glows_l2.py b/imap_processing/glows/l2/glows_l2.py index 41949f9ab..75daf3662 100644 --- a/imap_processing/glows/l2/glows_l2.py +++ b/imap_processing/glows/l2/glows_l2.py @@ -314,6 +314,7 @@ def return_good_times(flags: xr.DataArray, active_flags: NDArray) -> NDArray: """ if len(active_flags) != flags.shape[1]: print("Active flags don't matched expected length") + print(f"Active flags: {active_flags}") # A good time is where all the active flags are equal to one. # Here, we mask the active indices using active_flags, and then return the times From 79c5038ef272d46c22ac9b21534ce7b1df47155f Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Thu, 19 Feb 2026 10:20:38 -0700 Subject: [PATCH 2/3] Remove prints --- imap_processing/glows/l1b/glows_l1b_data.py | 1 - imap_processing/glows/l2/glows_l2.py | 1 - 2 files changed, 2 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 50f9cee2e..a5409f187 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -155,7 +155,6 @@ def __post_init__(self, pipeline_dataset: xr.Dataset) -> None: bool(pipeline_dataset[f"active_bad_time_flags_{name}"].values) for name in _time_flag_names ] - print(self.active_bad_time_flags) else: # Default: assume all bad-time flags are active self.active_bad_time_flags = [True] * FLAG_LENGTH diff --git a/imap_processing/glows/l2/glows_l2.py b/imap_processing/glows/l2/glows_l2.py index 75daf3662..41949f9ab 100644 --- a/imap_processing/glows/l2/glows_l2.py +++ b/imap_processing/glows/l2/glows_l2.py @@ -314,7 +314,6 @@ def return_good_times(flags: xr.DataArray, active_flags: NDArray) -> NDArray: """ if len(active_flags) != flags.shape[1]: print("Active flags don't matched expected length") - print(f"Active flags: {active_flags}") # A good time is where all the active flags are equal to one. # Here, we mask the active indices using active_flags, and then return the times From 210c33fd2bc7974cef3816ac394909b0d028805b Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Thu, 19 Feb 2026 12:51:51 -0700 Subject: [PATCH 3/3] Add test --- .../tests/glows/test_glows_l1b_data.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/imap_processing/tests/glows/test_glows_l1b_data.py b/imap_processing/tests/glows/test_glows_l1b_data.py index 477c3fdb6..d7f9114bb 100644 --- a/imap_processing/tests/glows/test_glows_l1b_data.py +++ b/imap_processing/tests/glows/test_glows_l1b_data.py @@ -4,12 +4,14 @@ import numpy as np import pytest +import xarray as xr from imap_processing.glows.l1b.glows_l1b import glows_l1b, glows_l1b_de from imap_processing.glows.l1b.glows_l1b_data import ( AncillaryParameters, DirectEventL1B, HistogramL1B, + PipelineSettings, ) from imap_processing.spice.time import met_to_ttj2000ns from imap_processing.tests.glows.conftest import mock_update_spice_parameters @@ -221,3 +223,61 @@ def test_validation_data_de( def test_deserialize_flags(flags, expected): output = HistogramL1B.deserialize_flags(flags) assert np.array_equal(output, expected) + + +def test_pipeline_settings_from_flattened_json(): + """PipelineSettings correctly reads flags from flattened JSON format. + + convert_json_to_dataset flattens nested dicts, so + active_bad_time_flags.is_night -> active_bad_time_flags_is_night. + PipelineSettings must reconstruct the ordered flag lists from these keys. + """ + data_vars = { + "active_bad_time_flags_is_pps_missing": ([], True), + "active_bad_time_flags_is_time_status_missing": ([], True), + "active_bad_time_flags_is_phase_missing": ([], True), + "active_bad_time_flags_is_spin_period_missing": ([], True), + "active_bad_time_flags_is_overexposed": ([], True), + "active_bad_time_flags_is_direct_event_non_monotonic": ([], True), + "active_bad_time_flags_is_night": ([], False), + "active_bad_time_flags_is_hv_test_in_progress": ([], True), + "active_bad_time_flags_is_test_pulse_in_progress": ([], True), + "active_bad_time_flags_is_memory_error_detected": ([], True), + "active_bad_time_flags_is_generated_on_ground": ([], True), + "active_bad_time_flags_is_beyond_daily_statistical_error": ( + [], + True, + ), + "active_bad_time_flags_is_temperature_std_dev_beyond_threshold": ( + [], + True, + ), + "active_bad_time_flags_is_hv_voltage_std_dev_beyond_threshold": ( + [], + True, + ), + "active_bad_time_flags_is_spin_period_std_dev_beyond_threshold": ( + [], + True, + ), + "active_bad_time_flags_is_pulse_length_std_dev_beyond_threshold": ( + [], + True, + ), + "active_bad_time_flags_is_spin_period_difference_beyond_threshold": ( + [], + False, + ), + "active_bad_angle_flags_is_close_to_uv_source": ([], True), + "active_bad_angle_flags_is_inside_excluded_region": ([], True), + "active_bad_angle_flags_is_excluded_by_instr_team": ([], True), + "active_bad_angle_flags_is_suspected_transient": ([], False), + } + settings = PipelineSettings(xr.Dataset(data_vars)) + + assert len(settings.active_bad_time_flags) == 17 + assert settings.active_bad_time_flags[6] is False # is_night + assert settings.active_bad_time_flags[16] is False # is_spin_period_diff + + assert len(settings.active_bad_angle_flags) == 4 + assert settings.active_bad_angle_flags[3] is False # is_suspected_transient