Add output threshold and myopic capacity config

temoa/_internal/table_writer.py

@@ -4,6 +4,7 @@
 
 from __future__ import annotations
 
+import math
 import sqlite3
 import sys
 from collections import defaultdict
@@ -64,6 +65,13 @@
     'output_storage_level',
 ]
 
+OUTPUT_THRESHOLD_DEFAULTS = {
+    'capacity': 1e-3,
+    'activity': 1e-3,
+    'emission': 1e-3,
+    'cost': 1e-2,
+}
+
 FLOW_SUMMARY_FILE_LOC = (
     resources.files('temoa.extensions.modeling_to_generate_alternatives')
     / 'make_flow_summary_table.sql'
@@ -74,9 +82,8 @@
 class TableWriter:
     con: sqlite3.Connection | None
 
-    def __init__(self, config: TemoaConfig, epsilon: float = 1e-5) -> None:
+    def __init__(self, config: TemoaConfig) -> None:
         self.config = config
-        self.epsilon = epsilon
         self.tech_sectors: dict[str, str] | None = None
         self.flow_register: dict[FI, dict[FlowType, float]] = {}
         self.emission_register: dict[EI, float] | None = None
@@ -92,6 +99,17 @@ def __init__(self, config: TemoaConfig, epsilon: float = 1e-5) -> None:
             logger.exception('Failed to connect to output database: %s', config.output_database)
             sys.exit(-1)
 
+        self.output_threshold_capacity = self._resolve_output_threshold('capacity')
+        self.output_threshold_activity = self._resolve_output_threshold('activity')
+        self.output_threshold_emission = self._resolve_output_threshold('emission')
+        self.output_threshold_cost = self._resolve_output_threshold('cost')
+        self.epsilon = min(
+            self.output_threshold_capacity,
+            self.output_threshold_activity,
+            self.output_threshold_emission,
+            self.output_threshold_cost,
+        )
+
         # Unit propagator for populating units in output tables (lazy init)
         self._unit_propagator: UnitPropagator | None = None
 
@@ -228,6 +246,36 @@ def _bulk_insert(self, table_name: str, records: list[dict[str, Any]]) -> None:
 
         self.connection.executemany(query, rows_to_insert)
 
+    @staticmethod
+    def _validate_threshold(
+        threshold_name: str,
+        threshold_value: float | int | None,
+    ) -> float | None:
+        """Validate output threshold inputs and normalize to float."""
+        if threshold_value is None:
+            return None
+        numeric_value = float(threshold_value)
+        if not math.isfinite(numeric_value):
+            raise ValueError(f'Output threshold "{threshold_name}" must be finite')
+        if numeric_value < 0:
+            raise ValueError(f'Output threshold "{threshold_name}" must be non-negative')
+        return numeric_value
+
+    def _resolve_output_threshold(self, threshold_type: str) -> float:
+        """Resolve threshold: TOML config if set, else hardcoded default."""
+        if threshold_type not in OUTPUT_THRESHOLD_DEFAULTS:
+            raise ValueError(f'Unknown output threshold type: {threshold_type}')
+
+        toml_key = f'output_threshold_{threshold_type}'
+        toml_value = self._validate_threshold(
+            toml_key,
+            getattr(self.config, toml_key, None),
+        )
+        if toml_value is not None:
+            return toml_value
+
+        return OUTPUT_THRESHOLD_DEFAULTS[threshold_type]
+
     def write_results(
         self,
         model: TemoaModel,
@@ -252,7 +300,11 @@ def write_results(
             else:
                 p_0 = None
 
-            e_costs, e_flows = poll_emissions(model=model, p_0=value(p_0))
+            e_costs, e_flows = poll_emissions(
+                model=model,
+                p_0=value(p_0),
+                epsilon=self.output_threshold_emission,
+            )
             self.emission_register = e_flows
             self.write_emissions(iteration=iteration)
 
@@ -278,7 +330,10 @@ def write_mm_results(self, model: TemoaModel, iteration: int) -> None:
             if not self.tech_sectors:
                 self._set_tech_sectors()
             self.write_objective(model, iteration=iteration)
-            _e_costs, e_flows = poll_emissions(model=model)
+            _e_costs, e_flows = poll_emissions(
+                model=model,
+                epsilon=self.output_threshold_emission,
+            )
             self.emission_register = e_flows
             self.write_emissions(iteration=iteration)
         finally:
@@ -400,7 +455,7 @@ def write_emissions(self, iteration: int | None = None) -> None:
         records = []
 
         for ei, val in self.emission_register.items():
-            if abs(val) < self.epsilon:
+            if abs(val) < self.output_threshold_emission:
                 continue
 
             row = {
@@ -425,7 +480,7 @@ def write_emissions(self, iteration: int | None = None) -> None:
         self.connection.commit()
 
     def write_capacity_tables(self, model: TemoaModel, iteration: int | None = None) -> None:
-        cap_data = poll_capacity_results(model=model)
+        cap_data = poll_capacity_results(model=model, epsilon=self.output_threshold_capacity)
         self._insert_capacity_results(cap_data=cap_data, iteration=iteration)
 
     def _insert_capacity_results(self, cap_data: CapData, iteration: int | None) -> None:
@@ -508,7 +563,7 @@ def write_flow_tables(self, iteration: int | None = None) -> None:
             sector = self.tech_sectors.get(fi.t)
 
             for flow_type, val in flows.items():
-                if abs(val) < self.epsilon:
+                if abs(val) < self.output_threshold_activity:
                     continue
 
                 table_name = map_flow_to_table.get(flow_type)
@@ -593,7 +648,7 @@ def _insert_summary_flow_results(
 
         records = []
         for (r, p, i, t, v, o), val in output_flows.items():
-            if abs(val) < self.epsilon:
+            if abs(val) < self.output_threshold_activity:
                 continue
             records.append(
                 {
@@ -643,7 +698,7 @@ def check_flow_balance(self, model: TemoaModel) -> bool:
         return all_good
 
     def calculate_flows(self, model: TemoaModel) -> dict[FI, dict[FlowType, float]]:
-        return poll_flow_results(model, self.epsilon)
+        return poll_flow_results(model, self.output_threshold_activity)
 
     def write_costs(
         self,
@@ -657,7 +712,7 @@ def write_costs(
         else:
             p_0 = min(model.time_optimize)
 
-        entries, exchange_entries = poll_cost_results(model, value(p_0), self.epsilon)
+        entries, exchange_entries = poll_cost_results(model, value(p_0), self.output_threshold_cost)
         self._insert_cost_results(entries, exchange_entries, emission_entries, iteration)
 
     def _insert_cost_results(
@@ -697,6 +752,18 @@ def _write_cost_rows(
 
         for r, p, t, v in sorted_keys:
             costs = entries[(r, p, t, v)]
+            row_values = (
+                costs.get(CostType.D_INVEST, 0),
+                costs.get(CostType.D_FIXED, 0),
+                costs.get(CostType.D_VARIABLE, 0),
+                costs.get(CostType.D_EMISS, 0),
+                costs.get(CostType.INVEST, 0),
+                costs.get(CostType.FIXED, 0),
+                costs.get(CostType.VARIABLE, 0),
+                costs.get(CostType.EMISS, 0),
+            )
+            if all(abs(val) < self.output_threshold_cost for val in row_values):
+                continue
             records.append(
                 {
                     'scenario': scenario,

temoa/core/config.py

@@ -65,6 +65,10 @@ def __init__(
         graphviz_output: bool = False,
         cycle_count_limit: int = 100,
         cycle_length_limit: int = 1,
+        output_threshold_capacity: float | None = None,
+        output_threshold_activity: float | None = None,
+        output_threshold_emission: float | None = None,
+        output_threshold_cost: float | None = None,
     ):
         if '-' in scenario:
             raise ValueError(
@@ -148,6 +152,10 @@ def __init__(
         self.plot_commodity_network = plot_commodity_network and self.source_trace
         self.graphviz_output = graphviz_output
         self.stochastic_config = stochastic_config
+        self.output_threshold_capacity = output_threshold_capacity
+        self.output_threshold_activity = output_threshold_activity
+        self.output_threshold_emission = output_threshold_emission
+        self.output_threshold_cost = output_threshold_cost
 
         # Cycle detection limits
         if not isinstance(cycle_count_limit, int) or cycle_count_limit < -1:

temoa/extensions/myopic/myopic_sequencer.py

@@ -64,7 +64,15 @@ class MyopicSequencer:
     ]
 
     def __init__(self, config: TemoaConfig | None):
-        self.capacity_epsilon = 1e-5
+        # Minimum capacity (MW) to carry forward between myopic periods.
+        # Configurable via [myopic] capacity_threshold in TOML.
+        default_cap_threshold = 1e-3
+        if config and config.myopic_inputs:
+            self.capacity_epsilon = config.myopic_inputs.get(
+                'capacity_threshold', default_cap_threshold
+            )
+        else:
+            self.capacity_epsilon = default_cap_threshold
         self.debugging = False
         self.optimization_periods: list[int] | None = None
         self.instance_queue: deque[MyopicIndex] = deque()  # a LIFO queue
@@ -387,7 +395,7 @@ def update_myopic_efficiency_table(self, myopic_index: MyopicIndex, prev_base: i
                 'DELETE FROM myopic_efficiency '
                 'WHERE (SELECT region, tech, vintage) '
                 '  NOT IN (SELECT region, tech, vintage FROM output_net_capacity '
-                '    WHERE period = ? AND scenario = ?) '
+                '    WHERE period = ? AND scenario = ? AND ABS(capacity) >= ?) '
                 'AND tech not in (SELECT tech FROM main.technology where unlim_cap > 0)'
             )
 
@@ -396,16 +404,20 @@ def update_myopic_efficiency_table(self, myopic_index: MyopicIndex, prev_base: i
                     'SELECT * FROM myopic_efficiency '
                     'WHERE (SELECT region, tech, vintage) '
                     '  NOT IN (SELECT region, tech, vintage FROM output_net_capacity '
-                    '    WHERE period = ? AND scenario = ?) '
+                    '    WHERE period = ? AND scenario = ? AND ABS(capacity) >= ?) '
                     'AND tech not in (SELECT tech FROM Technology where unlim_cap > 0)'
                 )
                 print('\n\n **** Removing these unused region-tech-vintage combos ****')
                 removals = self.cursor.execute(
-                    debug_query, (last_interval_end, self.config.scenario)
+                    debug_query,
+                    (last_interval_end, self.config.scenario, self.capacity_epsilon),
                 ).fetchall()
                 for i, removal in enumerate(removals):
                     print(f'{i}. Removing:  {removal}')
-            self.cursor.execute(delete_qry, (last_interval_end, self.config.scenario))
+            self.cursor.execute(
+                delete_qry,
+                (last_interval_end, self.config.scenario, self.capacity_epsilon),
+            )
             self.output_con.commit()
 
         # 2.  Add the new stuff now visible

temoa/tutorial_assets/config_sample.toml

@@ -92,6 +92,13 @@ save_lp_file = false
 # graphviz dot file and svg for network visualization (requires graphviz to be installed separately)
 graphviz_output = false
 
+# Optional output filtering thresholds (set to 0 to disable per category)
+# Precedence is: TOML value > internal defaults.
+output_threshold_capacity = 0.001
+output_threshold_activity = 0.001
+output_threshold_emission = 0.001
+output_threshold_cost = 0.01
+
 # ------------------------------------
 #          MODEL PARAMETERS
 #  these are specific to each model