diff --git a/imap_processing/cli.py b/imap_processing/cli.py index f44dce139..cece0fd92 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -739,7 +739,7 @@ def do_processing( datasets = [glows_l1b_de(input_dataset, conversion_table_dict)] if self.data_level == "l2": - science_files = dependencies.get_file_paths(source="glows") + science_files = dependencies.get_file_paths(source="glows", data_type="l1b") if len(science_files) != 1: raise ValueError( f"GLOWS L2 requires exactly one input science file, " diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 429173888..a5409f187 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -99,22 +99,65 @@ def __post_init__(self, pipeline_dataset: xr.Dataset) -> None: Dataset containing pipeline settings data variables. """ # Extract active bad-angle flags (default to all True if not present) + _angle_flag_names = [ + "is_close_to_uv_source", + "is_inside_excluded_region", + "is_excluded_by_instr_team", + "is_suspected_transient", + ] if "active_bad_angle_flags" in pipeline_dataset.data_vars: self.active_bad_angle_flags = list( pipeline_dataset["active_bad_angle_flags"].values ) + elif any( + f"active_bad_angle_flags_{n}" in pipeline_dataset.data_vars + for n in _angle_flag_names + ): + # Flattened format from convert_json_to_dataset + self.active_bad_angle_flags = [ + bool(pipeline_dataset[f"active_bad_angle_flags_{name}"].values) + for name in _angle_flag_names + ] else: # Default: all 4 bad-angle flags are active self.active_bad_angle_flags = [True, True, True, True] # Extract active bad-time flags (default to all True if not present) + _time_flag_names = [ + "is_pps_missing", + "is_time_status_missing", + "is_phase_missing", + "is_spin_period_missing", + "is_overexposed", + "is_direct_event_non_monotonic", + "is_night", + "is_hv_test_in_progress", + "is_test_pulse_in_progress", + "is_memory_error_detected", + "is_generated_on_ground", + "is_beyond_daily_statistical_error", + "is_temperature_std_dev_beyond_threshold", + "is_hv_voltage_std_dev_beyond_threshold", + "is_spin_period_std_dev_beyond_threshold", + "is_pulse_length_std_dev_beyond_threshold", + "is_spin_period_difference_beyond_threshold", + ] if "active_bad_time_flags" in pipeline_dataset.data_vars: self.active_bad_time_flags = list( pipeline_dataset["active_bad_time_flags"].values ) + elif any( + f"active_bad_time_flags_{n}" in pipeline_dataset.data_vars + for n in _time_flag_names + ): + # Flattened format from convert_json_to_dataset + self.active_bad_time_flags = [ + bool(pipeline_dataset[f"active_bad_time_flags_{name}"].values) + for name in _time_flag_names + ] else: # Default: assume all bad-time flags are active - self.active_bad_time_flags = [True] * 16 # Typical number of bad-time flags + self.active_bad_time_flags = [True] * FLAG_LENGTH # Extract sunrise/sunset offsets (default to 0.0 if not present) self.sunrise_offset = float(pipeline_dataset.get("sunrise_offset", 0.0)) diff --git a/imap_processing/tests/glows/test_glows_l1b_data.py b/imap_processing/tests/glows/test_glows_l1b_data.py index 477c3fdb6..d7f9114bb 100644 --- a/imap_processing/tests/glows/test_glows_l1b_data.py +++ b/imap_processing/tests/glows/test_glows_l1b_data.py @@ -4,12 +4,14 @@ import numpy as np import pytest +import xarray as xr from imap_processing.glows.l1b.glows_l1b import glows_l1b, glows_l1b_de from imap_processing.glows.l1b.glows_l1b_data import ( AncillaryParameters, DirectEventL1B, HistogramL1B, + PipelineSettings, ) from imap_processing.spice.time import met_to_ttj2000ns from imap_processing.tests.glows.conftest import mock_update_spice_parameters @@ -221,3 +223,61 @@ def test_validation_data_de( def test_deserialize_flags(flags, expected): output = HistogramL1B.deserialize_flags(flags) assert np.array_equal(output, expected) + + +def test_pipeline_settings_from_flattened_json(): + """PipelineSettings correctly reads flags from flattened JSON format. + + convert_json_to_dataset flattens nested dicts, so + active_bad_time_flags.is_night -> active_bad_time_flags_is_night. + PipelineSettings must reconstruct the ordered flag lists from these keys. + """ + data_vars = { + "active_bad_time_flags_is_pps_missing": ([], True), + "active_bad_time_flags_is_time_status_missing": ([], True), + "active_bad_time_flags_is_phase_missing": ([], True), + "active_bad_time_flags_is_spin_period_missing": ([], True), + "active_bad_time_flags_is_overexposed": ([], True), + "active_bad_time_flags_is_direct_event_non_monotonic": ([], True), + "active_bad_time_flags_is_night": ([], False), + "active_bad_time_flags_is_hv_test_in_progress": ([], True), + "active_bad_time_flags_is_test_pulse_in_progress": ([], True), + "active_bad_time_flags_is_memory_error_detected": ([], True), + "active_bad_time_flags_is_generated_on_ground": ([], True), + "active_bad_time_flags_is_beyond_daily_statistical_error": ( + [], + True, + ), + "active_bad_time_flags_is_temperature_std_dev_beyond_threshold": ( + [], + True, + ), + "active_bad_time_flags_is_hv_voltage_std_dev_beyond_threshold": ( + [], + True, + ), + "active_bad_time_flags_is_spin_period_std_dev_beyond_threshold": ( + [], + True, + ), + "active_bad_time_flags_is_pulse_length_std_dev_beyond_threshold": ( + [], + True, + ), + "active_bad_time_flags_is_spin_period_difference_beyond_threshold": ( + [], + False, + ), + "active_bad_angle_flags_is_close_to_uv_source": ([], True), + "active_bad_angle_flags_is_inside_excluded_region": ([], True), + "active_bad_angle_flags_is_excluded_by_instr_team": ([], True), + "active_bad_angle_flags_is_suspected_transient": ([], False), + } + settings = PipelineSettings(xr.Dataset(data_vars)) + + assert len(settings.active_bad_time_flags) == 17 + assert settings.active_bad_time_flags[6] is False # is_night + assert settings.active_bad_time_flags[16] is False # is_spin_period_diff + + assert len(settings.active_bad_angle_flags) == 4 + assert settings.active_bad_angle_flags[3] is False # is_suspected_transient