diff --git a/temoa/_internal/table_writer.py b/temoa/_internal/table_writer.py index c671abcc..ca59ec99 100644 --- a/temoa/_internal/table_writer.py +++ b/temoa/_internal/table_writer.py @@ -4,6 +4,7 @@ from __future__ import annotations +import math import sqlite3 import sys from collections import defaultdict @@ -64,6 +65,13 @@ 'output_storage_level', ] +OUTPUT_THRESHOLD_DEFAULTS = { + 'capacity': 1e-3, + 'activity': 1e-3, + 'emission': 1e-3, + 'cost': 1e-2, +} + FLOW_SUMMARY_FILE_LOC = ( resources.files('temoa.extensions.modeling_to_generate_alternatives') / 'make_flow_summary_table.sql' @@ -74,9 +82,8 @@ class TableWriter: con: sqlite3.Connection | None - def __init__(self, config: TemoaConfig, epsilon: float = 1e-5) -> None: + def __init__(self, config: TemoaConfig) -> None: self.config = config - self.epsilon = epsilon self.tech_sectors: dict[str, str] | None = None self.flow_register: dict[FI, dict[FlowType, float]] = {} self.emission_register: dict[EI, float] | None = None @@ -92,6 +99,17 @@ def __init__(self, config: TemoaConfig, epsilon: float = 1e-5) -> None: logger.exception('Failed to connect to output database: %s', config.output_database) sys.exit(-1) + self.output_threshold_capacity = self._resolve_output_threshold('capacity') + self.output_threshold_activity = self._resolve_output_threshold('activity') + self.output_threshold_emission = self._resolve_output_threshold('emission') + self.output_threshold_cost = self._resolve_output_threshold('cost') + self.epsilon = min( + self.output_threshold_capacity, + self.output_threshold_activity, + self.output_threshold_emission, + self.output_threshold_cost, + ) + # Unit propagator for populating units in output tables (lazy init) self._unit_propagator: UnitPropagator | None = None @@ -228,6 +246,36 @@ def _bulk_insert(self, table_name: str, records: list[dict[str, Any]]) -> None: self.connection.executemany(query, rows_to_insert) + @staticmethod + def _validate_threshold( + threshold_name: str, + threshold_value: float | int | None, + ) -> float | None: + """Validate output threshold inputs and normalize to float.""" + if threshold_value is None: + return None + numeric_value = float(threshold_value) + if not math.isfinite(numeric_value): + raise ValueError(f'Output threshold "{threshold_name}" must be finite') + if numeric_value < 0: + raise ValueError(f'Output threshold "{threshold_name}" must be non-negative') + return numeric_value + + def _resolve_output_threshold(self, threshold_type: str) -> float: + """Resolve threshold: TOML config if set, else hardcoded default.""" + if threshold_type not in OUTPUT_THRESHOLD_DEFAULTS: + raise ValueError(f'Unknown output threshold type: {threshold_type}') + + toml_key = f'output_threshold_{threshold_type}' + toml_value = self._validate_threshold( + toml_key, + getattr(self.config, toml_key, None), + ) + if toml_value is not None: + return toml_value + + return OUTPUT_THRESHOLD_DEFAULTS[threshold_type] + def write_results( self, model: TemoaModel, @@ -252,7 +300,11 @@ def write_results( else: p_0 = None - e_costs, e_flows = poll_emissions(model=model, p_0=value(p_0)) + e_costs, e_flows = poll_emissions( + model=model, + p_0=value(p_0), + epsilon=self.output_threshold_emission, + ) self.emission_register = e_flows self.write_emissions(iteration=iteration) @@ -278,7 +330,10 @@ def write_mm_results(self, model: TemoaModel, iteration: int) -> None: if not self.tech_sectors: self._set_tech_sectors() self.write_objective(model, iteration=iteration) - _e_costs, e_flows = poll_emissions(model=model) + _e_costs, e_flows = poll_emissions( + model=model, + epsilon=self.output_threshold_emission, + ) self.emission_register = e_flows self.write_emissions(iteration=iteration) finally: @@ -400,7 +455,7 @@ def write_emissions(self, iteration: int | None = None) -> None: records = [] for ei, val in self.emission_register.items(): - if abs(val) < self.epsilon: + if abs(val) < self.output_threshold_emission: continue row = { @@ -425,7 +480,7 @@ def write_emissions(self, iteration: int | None = None) -> None: self.connection.commit() def write_capacity_tables(self, model: TemoaModel, iteration: int | None = None) -> None: - cap_data = poll_capacity_results(model=model) + cap_data = poll_capacity_results(model=model, epsilon=self.output_threshold_capacity) self._insert_capacity_results(cap_data=cap_data, iteration=iteration) def _insert_capacity_results(self, cap_data: CapData, iteration: int | None) -> None: @@ -508,7 +563,7 @@ def write_flow_tables(self, iteration: int | None = None) -> None: sector = self.tech_sectors.get(fi.t) for flow_type, val in flows.items(): - if abs(val) < self.epsilon: + if abs(val) < self.output_threshold_activity: continue table_name = map_flow_to_table.get(flow_type) @@ -593,7 +648,7 @@ def _insert_summary_flow_results( records = [] for (r, p, i, t, v, o), val in output_flows.items(): - if abs(val) < self.epsilon: + if abs(val) < self.output_threshold_activity: continue records.append( { @@ -643,7 +698,7 @@ def check_flow_balance(self, model: TemoaModel) -> bool: return all_good def calculate_flows(self, model: TemoaModel) -> dict[FI, dict[FlowType, float]]: - return poll_flow_results(model, self.epsilon) + return poll_flow_results(model, self.output_threshold_activity) def write_costs( self, @@ -657,7 +712,7 @@ def write_costs( else: p_0 = min(model.time_optimize) - entries, exchange_entries = poll_cost_results(model, value(p_0), self.epsilon) + entries, exchange_entries = poll_cost_results(model, value(p_0), self.output_threshold_cost) self._insert_cost_results(entries, exchange_entries, emission_entries, iteration) def _insert_cost_results( @@ -697,6 +752,18 @@ def _write_cost_rows( for r, p, t, v in sorted_keys: costs = entries[(r, p, t, v)] + row_values = ( + costs.get(CostType.D_INVEST, 0), + costs.get(CostType.D_FIXED, 0), + costs.get(CostType.D_VARIABLE, 0), + costs.get(CostType.D_EMISS, 0), + costs.get(CostType.INVEST, 0), + costs.get(CostType.FIXED, 0), + costs.get(CostType.VARIABLE, 0), + costs.get(CostType.EMISS, 0), + ) + if all(abs(val) < self.output_threshold_cost for val in row_values): + continue records.append( { 'scenario': scenario, diff --git a/temoa/core/config.py b/temoa/core/config.py index d61ee04e..962893f8 100644 --- a/temoa/core/config.py +++ b/temoa/core/config.py @@ -65,6 +65,10 @@ def __init__( graphviz_output: bool = False, cycle_count_limit: int = 100, cycle_length_limit: int = 1, + output_threshold_capacity: float | None = None, + output_threshold_activity: float | None = None, + output_threshold_emission: float | None = None, + output_threshold_cost: float | None = None, ): if '-' in scenario: raise ValueError( @@ -148,6 +152,10 @@ def __init__( self.plot_commodity_network = plot_commodity_network and self.source_trace self.graphviz_output = graphviz_output self.stochastic_config = stochastic_config + self.output_threshold_capacity = output_threshold_capacity + self.output_threshold_activity = output_threshold_activity + self.output_threshold_emission = output_threshold_emission + self.output_threshold_cost = output_threshold_cost # Cycle detection limits if not isinstance(cycle_count_limit, int) or cycle_count_limit < -1: diff --git a/temoa/extensions/myopic/myopic_sequencer.py b/temoa/extensions/myopic/myopic_sequencer.py index f4c5d943..4637824a 100644 --- a/temoa/extensions/myopic/myopic_sequencer.py +++ b/temoa/extensions/myopic/myopic_sequencer.py @@ -64,7 +64,15 @@ class MyopicSequencer: ] def __init__(self, config: TemoaConfig | None): - self.capacity_epsilon = 1e-5 + # Minimum capacity (MW) to carry forward between myopic periods. + # Configurable via [myopic] capacity_threshold in TOML. + default_cap_threshold = 1e-3 + if config and config.myopic_inputs: + self.capacity_epsilon = config.myopic_inputs.get( + 'capacity_threshold', default_cap_threshold + ) + else: + self.capacity_epsilon = default_cap_threshold self.debugging = False self.optimization_periods: list[int] | None = None self.instance_queue: deque[MyopicIndex] = deque() # a LIFO queue @@ -387,7 +395,7 @@ def update_myopic_efficiency_table(self, myopic_index: MyopicIndex, prev_base: i 'DELETE FROM myopic_efficiency ' 'WHERE (SELECT region, tech, vintage) ' ' NOT IN (SELECT region, tech, vintage FROM output_net_capacity ' - ' WHERE period = ? AND scenario = ?) ' + ' WHERE period = ? AND scenario = ? AND ABS(capacity) >= ?) ' 'AND tech not in (SELECT tech FROM main.technology where unlim_cap > 0)' ) @@ -396,16 +404,20 @@ def update_myopic_efficiency_table(self, myopic_index: MyopicIndex, prev_base: i 'SELECT * FROM myopic_efficiency ' 'WHERE (SELECT region, tech, vintage) ' ' NOT IN (SELECT region, tech, vintage FROM output_net_capacity ' - ' WHERE period = ? AND scenario = ?) ' + ' WHERE period = ? AND scenario = ? AND ABS(capacity) >= ?) ' 'AND tech not in (SELECT tech FROM Technology where unlim_cap > 0)' ) print('\n\n **** Removing these unused region-tech-vintage combos ****') removals = self.cursor.execute( - debug_query, (last_interval_end, self.config.scenario) + debug_query, + (last_interval_end, self.config.scenario, self.capacity_epsilon), ).fetchall() for i, removal in enumerate(removals): print(f'{i}. Removing: {removal}') - self.cursor.execute(delete_qry, (last_interval_end, self.config.scenario)) + self.cursor.execute( + delete_qry, + (last_interval_end, self.config.scenario, self.capacity_epsilon), + ) self.output_con.commit() # 2. Add the new stuff now visible diff --git a/temoa/tutorial_assets/config_sample.toml b/temoa/tutorial_assets/config_sample.toml index 4cf8d767..e2c5573b 100644 --- a/temoa/tutorial_assets/config_sample.toml +++ b/temoa/tutorial_assets/config_sample.toml @@ -92,6 +92,13 @@ save_lp_file = false # graphviz dot file and svg for network visualization (requires graphviz to be installed separately) graphviz_output = false +# Optional output filtering thresholds (set to 0 to disable per category) +# Precedence is: TOML value > internal defaults. +output_threshold_capacity = 0.001 +output_threshold_activity = 0.001 +output_threshold_emission = 0.001 +output_threshold_cost = 0.01 + # ------------------------------------ # MODEL PARAMETERS # these are specific to each model