diff --git a/temoa/components/limits.py b/temoa/components/limits.py index 3f55985a..ff806e28 100644 --- a/temoa/components/limits.py +++ b/temoa/components/limits.py @@ -23,6 +23,8 @@ from temoa.components.utils import Operator, get_variable_efficiency, operator_expression if TYPE_CHECKING: + from pyomo.core import Expression + from temoa.core.model import TemoaModel from temoa.types import ExprLike, Period, Region, Technology, Vintage from temoa.types.core_types import Commodity, Season, TimeOfDay @@ -197,6 +199,20 @@ def limit_degrowth_new_capacity_delta_indices( return indices +def limit_annual_capacity_factor_indices( + model: TemoaModel, +) -> set[tuple[Region, Period, Technology, Vintage, Commodity, str]]: + indices = { + (r, p, t, v, o, op) + for r, t, v, o, op in model.limit_annual_capacity_factor_constraint_rtvo + for _r in geography.gather_group_regions(model, r) + for _t in technology.gather_group_techs(model, t) + for p in model.time_optimize + if o in model.process_outputs.get((_r, p, _t, v), []) + } + return indices + + # ============================================================================ # PYOMO CONSTRAINT RULES # ============================================================================ @@ -409,7 +425,7 @@ def limit_capacity_share_constraint( def limit_new_capacity_share_constraint( - model: TemoaModel, r: Region, p: Period, g1: Technology, g2: Technology, op: str + model: TemoaModel, r: Region, g1: Technology, g2: Technology, v: Vintage, op: str ) -> ExprLike: r""" The limit_new_capacity_share constraint limits the share of new capacity @@ -420,21 +436,21 @@ def limit_new_capacity_share_constraint( sub_group = technology.gather_group_techs(model, g1) sub_new_cap = quicksum( - model.v_new_capacity[_r, _t, p] + model.v_new_capacity[_r, _t, v] for _t in sub_group for _r in regions - if (_r, _t, cast('Vintage', p)) in model.process_periods + if (_r, _t, v) in model.process_periods ) super_group = technology.gather_group_techs(model, g2) super_new_cap = quicksum( - model.v_new_capacity[_r, _t, p] + model.v_new_capacity[_r, _t, v] for _t in super_group for _r in regions - if (_r, _t, cast('Vintage', p)) in model.process_periods + if (_r, _t, v) in model.process_periods ) - share_lim = value(model.limit_new_capacity_share[r, p, g1, g2, op]) + share_lim = value(model.limit_new_capacity_share[r, g1, g2, v, op]) expr = operator_expression(sub_new_cap, Operator(op), share_lim * super_new_cap) if isinstance(expr, bool): return Constraint.Skip @@ -442,11 +458,11 @@ def limit_new_capacity_share_constraint( def limit_annual_capacity_factor_constraint( - model: TemoaModel, r: Region, p: Period, t: Technology, o: Commodity, op: str + model: TemoaModel, r: Region, p: Period, t: Technology, v: Vintage, o: Commodity, op: str ) -> ExprLike: r""" The limit_annual_capacity_factor sets an upper bound on the annual capacity factor - from a specific technology. The first portion of the constraint pertains to + from a specific process. The first portion of the constraint pertains to technologies with variable output at the time slice level, and the second portion pertains to technologies with constant annual output belonging to the :code:`tech_annual` set. @@ -454,49 +470,50 @@ def limit_annual_capacity_factor_constraint( .. math:: :label: limit_annual_capacity_factor - \sum_{S,D,I,V,O} \textbf{FO}_{r, p, s, d, i, t, v, o} \le LIMACF_{r, p, t} \cdot - \textbf{CAPAVL}_{r, p, t} \cdot \text{C2A}_{r, t} + \sum_{S,D,I} \textbf{FO}_{r, p, s, d, i, t, v, o} \le LIMACF_{r, t, v, o} \cdot + \textbf{CAP}_{r, p, t, v} \cdot \text{C2A}_{r, t} - \forall \{r, p, t \notin T^{a}, o\} \in \Theta_{\text{limit\_annual\_capacity\_factor}} + \forall \{r, t \notin T^{a}, v, o\} \in \Theta_{\text{limit\_annual\_capacity\_factor}} - \\\sum_{I,V,O} \textbf{FOA}_{r, p, i, t, v, o} \ge LIMACF_{r, p, t} \cdot - \textbf{CAPAVL}_{r, p, t} \cdot \text{C2A}_{r, t} + \\\sum_{I} \textbf{FOA}_{r, p, i, t, v, o} \ge LIMACF_{r, t, v, o} \cdot + \textbf{CAP}_{r, p, t, v} \cdot \text{C2A}_{r, t} - \forall \{r, p, t \in T^{a}, o\} \in \Theta_{\text{limit\_annual\_capacity\_factor}} + \forall \{r, t \in T^{a}, v, o\} \in \Theta_{\text{limit\_annual\_capacity\_factor}} """ # r can be an individual region (r='US'), or a combination of regions separated by plus # (r='Mexico+US+Canada'), or 'global'. # if r == 'global', the constraint is system-wide regions = geography.gather_group_regions(model, r) - # we need to screen here because it is possible that the restriction extends beyond the - # lifetime of any vintage of the tech... - if all((_r, p, t) not in model.v_capacity_available_by_period_and_tech for _r in regions): - return Constraint.Skip + techs = technology.gather_group_techs(model, t) - if t not in model.tech_annual: - activity_rpt = quicksum( - model.v_flow_out[_r, p, s, d, S_i, t, S_v, o] - for _r in regions - for S_v in model.process_vintages.get((_r, p, t), []) - for S_i in model.process_inputs[_r, p, t, S_v] - for s in model.time_season[p] - for d in model.time_of_day - ) + if TYPE_CHECKING: + activity_rptvo = cast('Expression', 0) else: - activity_rpt = quicksum( - model.v_flow_out_annual[_r, p, S_i, t, S_v, o] - for _r in regions - for S_v in model.process_vintages.get((_r, p, t), []) - for S_i in model.process_inputs[_r, p, t, S_v] - ) + activity_rptvo = 0 + for _t in techs: + if _t not in model.tech_annual: + activity_rptvo += quicksum( + model.v_flow_out[_r, p, s, d, S_i, _t, v, o] + for _r in regions + for S_i in model.process_inputs_by_output.get((_r, p, _t, v, o), []) + for s in model.time_season[p] + for d in model.time_of_day + ) + else: + activity_rptvo += quicksum( + model.v_flow_out_annual[_r, p, S_i, _t, v, o] + for _r in regions + for S_i in model.process_inputs_by_output.get((_r, p, _t, v, o), []) + ) - possible_activity_rpt = quicksum( - model.v_capacity_available_by_period_and_tech[_r, p, t] - * value(model.capacity_to_activity[_r, t]) + possible_activity_rptvo = quicksum( + model.v_capacity[_r, p, _t, v] * value(model.capacity_to_activity[_r, _t]) for _r in regions + for _t in techs + if v in model.process_vintages.get((_r, p, _t), []) ) - annual_cf = value(model.limit_annual_capacity_factor[r, p, t, o, op]) - expr = operator_expression(activity_rpt, Operator(op), annual_cf * possible_activity_rpt) + annual_cf = value(model.limit_annual_capacity_factor[r, t, v, o, op]) + expr = operator_expression(activity_rptvo, Operator(op), annual_cf * possible_activity_rptvo) # in the case that there is nothing to sum, skip if isinstance(expr, bool): # an empty list was generated return Constraint.Skip @@ -531,35 +548,48 @@ def limit_seasonal_capacity_factor_constraint( # (r='Mexico+US+Canada'), or 'global'. # if r == 'global', the constraint is system-wide regions = geography.gather_group_regions(model, r) + techs = technology.gather_group_techs(model, t) + # we need to screen here because it is possible that the restriction extends beyond the # lifetime of any vintage of the tech... - if all((_r, p, t) not in model.v_capacity_available_by_period_and_tech for _r in regions): + if all( + (_r, p, _t) not in model.v_capacity_available_by_period_and_tech + for _r in regions + for _t in techs + ): return Constraint.Skip - if t not in model.tech_annual: - activity_rpst = quicksum( - model.v_flow_out[_r, p, s, d, S_i, t, S_v, S_o] - for _r in regions - for S_v in model.process_vintages[_r, p, t] - for S_i in model.process_inputs[_r, p, t, S_v] - for S_o in model.process_outputs_by_input[_r, p, t, S_v, S_i] - for d in model.time_of_day - ) + if TYPE_CHECKING: + activity_rpst = cast('Expression', 0) else: - activity_rpst = quicksum( - model.v_flow_out_annual[_r, p, S_i, t, S_v, S_o] - * model.segment_fraction_per_season[p, s] - for _r in regions - for S_v in model.process_vintages[_r, p, t] - for S_i in model.process_inputs[_r, p, t, S_v] - for S_o in model.process_outputs_by_input[_r, p, t, S_v, S_i] - ) + activity_rpst = 0 + for _t in techs: + if _t not in model.tech_annual: + activity_rpst += quicksum( + model.v_flow_out[_r, p, s, d, S_i, _t, S_v, S_o] + for _r in regions + for S_v in model.process_vintages.get((_r, p, _t), []) + for S_i in model.process_inputs.get((_r, p, _t, S_v), []) + for S_o in model.process_outputs_by_input.get((_r, p, _t, S_v, S_i), []) + for d in model.time_of_day + ) + else: + activity_rpst += quicksum( + model.v_flow_out_annual[_r, p, S_i, _t, S_v, S_o] + * model.segment_fraction_per_season[p, s] + for _r in regions + for S_v in model.process_vintages.get((_r, p, _t), []) + for S_i in model.process_inputs.get((_r, p, _t, S_v), []) + for S_o in model.process_outputs_by_input.get((_r, p, _t, S_v, S_i), []) + ) possible_activity_rpst = quicksum( - model.v_capacity_available_by_period_and_tech[_r, p, t] - * value(model.capacity_to_activity[_r, t]) + model.v_capacity_available_by_period_and_tech[_r, p, _t] + * value(model.capacity_to_activity[_r, _t]) * value(model.segment_fraction_per_season[p, s]) for _r in regions + for _t in techs + if (_r, p, _t) in model.v_capacity_available_by_period_and_tech ) seasonal_cf = value(model.limit_seasonal_capacity_factor[r, p, s, t, op]) expr = operator_expression(activity_rpst, Operator(op), seasonal_cf * possible_activity_rpst) @@ -1342,24 +1372,26 @@ def limit_activity_constraint( def limit_new_capacity_constraint( - model: TemoaModel, r: Region, p: Period, t: Technology, op: str + model: TemoaModel, r: Region, t: Technology, v: Vintage, op: str ) -> ExprLike: r""" The limit_new_capacity constraint sets a limit on the newly installed capacity of a - given technology or group in a given year. Note that the indices for these constraints are - region, period and tech. + given technology or group in a given vintage year. .. math:: :label: limit_new_capacity - \textbf{NCAP}_{r, t, v} \le LNC_{r, p, t} - - \text{where }v=p + \textbf{NCAP}_{r, t, v} \le LNC_{r, t, v} """ regions = geography.gather_group_regions(model, r) techs = technology.gather_group_techs(model, t) - cap_lim = value(model.limit_new_capacity[r, p, t, op]) - new_cap = quicksum(model.v_new_capacity[_r, _t, p] for _t in techs for _r in regions) + cap_lim = value(model.limit_new_capacity[r, t, v, op]) + new_cap = quicksum( + model.v_new_capacity[_r, _t, v] + for _t in techs + for _r in regions + if (_r, _t, v) in model.process_periods + ) expr = operator_expression(new_cap, Operator(op), cap_lim) if isinstance(expr, bool): return Constraint.Skip diff --git a/temoa/core/model.py b/temoa/core/model.py index bd96ab8f..88884cad 100755 --- a/temoa/core/model.py +++ b/temoa/core/model.py @@ -583,13 +583,13 @@ def __init__(self, *args: object, **kwargs: object) -> None: ) self.limit_capacity = Param(self.limit_capacity_constraint_rpt) - self.limit_new_capacity_constraint_rpt = Set( + self.limit_new_capacity_constraint_rtv = Set( within=self.regional_global_indices - * self.time_optimize * self.tech_or_group + * self.vintage_optimize * self.operator ) - self.limit_new_capacity = Param(self.limit_new_capacity_constraint_rpt) + self.limit_new_capacity = Param(self.limit_new_capacity_constraint_rtv) self.limit_resource_constraint_rt = Set( within=self.regional_global_indices * self.tech_or_group * self.operator @@ -608,22 +608,22 @@ def __init__(self, *args: object, **kwargs: object) -> None: within=self.regional_global_indices * self.time_optimize * self.time_season_all - * self.tech_all + * self.tech_or_group * self.operator ) self.limit_seasonal_capacity_factor = Param( self.limit_seasonal_capacity_factor_constraint_rpst, validate=validate_0to1 ) - self.limit_annual_capacity_factor_constraint_rpto = Set( + self.limit_annual_capacity_factor_constraint_rtvo = Set( within=self.regional_global_indices - * self.time_optimize - * self.tech_all + * self.tech_or_group + * self.vintage_all * self.commodity_carrier * self.operator ) self.limit_annual_capacity_factor = Param( - self.limit_annual_capacity_factor_constraint_rpto, validate=validate_0to1 + self.limit_annual_capacity_factor_constraint_rtvo, validate=validate_0to1 ) self.limit_growth_capacity = Param( @@ -673,14 +673,14 @@ def __init__(self, *args: object, **kwargs: object) -> None: ) self.limit_activity_share = Param(self.limit_activity_share_constraint_rpgg) - self.limit_new_capacity_share_constraint_rpgg = Set( + self.limit_new_capacity_share_constraint_rggv = Set( within=self.regional_global_indices - * self.time_optimize * self.tech_or_group * self.tech_or_group + * self.vintage_optimize * self.operator ) - self.limit_new_capacity_share = Param(self.limit_new_capacity_share_constraint_rpgg) + self.limit_new_capacity_share = Param(self.limit_new_capacity_share_constraint_rggv) # This set works for all storage-related constraints self.storage_constraints_rpsdtv = Set( @@ -1049,7 +1049,7 @@ def __init__(self, *args: object, **kwargs: object) -> None: ) self.limit_new_capacity_constraint = Constraint( - self.limit_new_capacity_constraint_rpt, rule=limits.limit_new_capacity_constraint + self.limit_new_capacity_constraint_rtv, rule=limits.limit_new_capacity_constraint ) self.limit_capacity_share_constraint = Constraint( @@ -1061,7 +1061,7 @@ def __init__(self, *args: object, **kwargs: object) -> None: ) self.limit_new_capacity_share_constraint = Constraint( - self.limit_new_capacity_share_constraint_rpgg, + self.limit_new_capacity_share_constraint_rggv, rule=limits.limit_new_capacity_share_constraint, ) @@ -1073,8 +1073,11 @@ def __init__(self, *args: object, **kwargs: object) -> None: self.limit_resource_constraint_rt, rule=limits.limit_resource_constraint ) + self.limit_annual_capacity_factor_constraint_rptvo = Set( + dimen=6, initialize=limits.limit_annual_capacity_factor_indices + ) self.limit_annual_capacity_factor_constraint = Constraint( - self.limit_annual_capacity_factor_constraint_rpto, + self.limit_annual_capacity_factor_constraint_rptvo, rule=limits.limit_annual_capacity_factor_constraint, ) diff --git a/temoa/data_io/component_manifest.py b/temoa/data_io/component_manifest.py index 37efe764..75789f63 100644 --- a/temoa/data_io/component_manifest.py +++ b/temoa/data_io/component_manifest.py @@ -595,59 +595,76 @@ def build_manifest(model: TemoaModel) -> list[LoadItem]: component=model.limit_capacity, table='limit_capacity', columns=['region', 'period', 'tech_or_group', 'operator', 'capacity'], + validator_name='viable_rpt', + validation_map=(0, 1, 2), is_table_required=False, ), LoadItem( component=model.limit_new_capacity, table='limit_new_capacity', - columns=['region', 'period', 'tech_or_group', 'operator', 'new_cap'], + columns=['region', 'tech_or_group', 'vintage', 'operator', 'new_cap'], + validator_name='viable_rtv', + validation_map=(0, 1, 2), + is_period_filtered=False, is_table_required=False, ), LoadItem( component=model.limit_capacity_share, table='limit_capacity_share', columns=['region', 'period', 'sub_group', 'super_group', 'operator', 'share'], + validator_name='viable_rpt', + validation_map=(0, 1, 2), is_table_required=False, ), LoadItem( component=model.limit_new_capacity_share, table='limit_new_capacity_share', - columns=['region', 'period', 'sub_group', 'super_group', 'operator', 'share'], + columns=['region', 'sub_group', 'super_group', 'vintage', 'operator', 'share'], + validator_name='viable_rtv', + validation_map=(0, 1, 3), + is_period_filtered=False, is_table_required=False, ), LoadItem( component=model.limit_activity, table='limit_activity', columns=['region', 'period', 'tech_or_group', 'operator', 'activity'], + validator_name='viable_rpt', + validation_map=(0, 1, 2), is_table_required=False, ), LoadItem( component=model.limit_activity_share, table='limit_activity_share', columns=['region', 'period', 'sub_group', 'super_group', 'operator', 'share'], + validator_name='viable_rpt', + validation_map=(0, 1, 2), is_table_required=False, ), LoadItem( component=model.limit_resource, table='limit_resource', columns=['region', 'tech_or_group', 'operator', 'cum_act'], + validator_name='viable_rt', + validation_map=(0, 1), is_period_filtered=False, is_table_required=False, ), LoadItem( component=model.limit_seasonal_capacity_factor, table='limit_seasonal_capacity_factor', - columns=['region', 'period', 'season', 'tech', 'operator', 'factor'], - validator_name='viable_rt', - validation_map=(0, 3), + columns=['region', 'period', 'season', 'tech_or_group', 'operator', 'factor'], + validator_name='viable_rpt', + validation_map=(0, 1, 3), is_table_required=False, ), LoadItem( component=model.limit_annual_capacity_factor, table='limit_annual_capacity_factor', - columns=['region', 'period', 'tech', 'output_comm', 'operator', 'factor'], - validator_name='viable_rpto', + columns=['region', 'tech_or_group', 'vintage', 'output_comm', 'operator', 'factor'], + validator_name='viable_rtvo', validation_map=(0, 1, 2, 3), + is_period_filtered=False, is_table_required=False, ), LoadItem( diff --git a/temoa/data_io/hybrid_loader.py b/temoa/data_io/hybrid_loader.py index 39cec6fd..faf3864c 100644 --- a/temoa/data_io/hybrid_loader.py +++ b/temoa/data_io/hybrid_loader.py @@ -106,6 +106,8 @@ def __init__(self, db_connection: Connection, config: TemoaConfig) -> None: self.viable_output_comms: ViableSet | None = None self.viable_vintages: ViableSet | None = None self.viable_ritvo: ViableSet | None = None + self.viable_rtvo: ViableSet | None = None + self.viable_rpt: ViableSet | None = None self.viable_rpto: ViableSet | None = None self.viable_rtv: ViableSet | None = None self.viable_rt: ViableSet | None = None @@ -246,9 +248,6 @@ def create_data_dict(self, myopic_index: MyopicIndex | None = None) -> dict[str, raw_data = item.fallback_data filtered_data = self._filter_data(raw_data, item, use_raw_data) - if not filtered_data: - continue - if len(filtered_data) < len(raw_data): ignored_count = len(raw_data) - len(filtered_data) msg = '%d values for %s failed to validate and were ignored.' @@ -457,8 +456,17 @@ def _build_efficiency_dataset( if not self.manager: raise RuntimeError('Source trace manager not initialized for filtering.') - filts = self.manager.build_filters() + # Build viable sets for tech_or_group + # Create a dictionary from the tech_group_members table + tech_groups = defaultdict(set) + if self.table_exists('tech_group_member'): + for group_name, tech in cur.execute('SELECT group_name, tech FROM tech_group_member'): + tech_groups[tech].add(group_name) + + filts = self.manager.build_filters(tech_groups) self.viable_ritvo = filts['ritvo'] + self.viable_rtvo = filts['rtvo'] + self.viable_rpt = filts['rpt'] self.viable_rtv = filts['rtv'] self.viable_rt = filts['rt'] self.viable_rpit = filts['rpit'] @@ -936,13 +944,13 @@ def load_param_idx_sets(self, data: dict[str, object]) -> dict[str, object]: ), model.limit_activity_share.name: model.limit_activity_share_constraint_rpgg.name, model.limit_annual_capacity_factor.name: ( - model.limit_annual_capacity_factor_constraint_rpto.name + model.limit_annual_capacity_factor_constraint_rtvo.name ), model.limit_capacity.name: model.limit_capacity_constraint_rpt.name, model.limit_capacity_share.name: model.limit_capacity_share_constraint_rpgg.name, - model.limit_new_capacity.name: model.limit_new_capacity_constraint_rpt.name, + model.limit_new_capacity.name: model.limit_new_capacity_constraint_rtv.name, model.limit_new_capacity_share.name: ( - model.limit_new_capacity_share_constraint_rpgg.name + model.limit_new_capacity_share_constraint_rggv.name ), model.limit_resource.name: model.limit_resource_constraint_rt.name, model.limit_storage_fraction.name: model.limit_storage_fraction_constraint_rpsdtv.name, diff --git a/temoa/db_schema/temoa_schema_v3_1.sql b/temoa/db_schema/temoa_schema_v3_1.sql index b91d03ce..303c7472 100644 --- a/temoa/db_schema/temoa_schema_v3_1.sql +++ b/temoa/db_schema/temoa_schema_v3_1.sql @@ -516,17 +516,16 @@ CREATE TABLE IF NOT EXISTS LimitActivityShare CREATE TABLE IF NOT EXISTS LimitAnnualCapacityFactor ( region TEXT, - period INTEGER + tech_or_group TEXT, + vintage INTEGER REFERENCES TimePeriod (period), - tech TEXT - REFERENCES Technology (tech), output_comm TEXT REFERENCES Commodity (name), operator TEXT NOT NULL DEFAULT "le" REFERENCES Operator (operator), factor REAL, notes TEXT, - PRIMARY KEY (region, period, tech, output_comm, operator), + PRIMARY KEY (region, tech_or_group, vintage, output_comm, operator), CHECK (factor >= 0 AND factor <= 1) ); CREATE TABLE IF NOT EXISTS LimitCapacity @@ -558,28 +557,28 @@ CREATE TABLE IF NOT EXISTS LimitCapacityShare CREATE TABLE IF NOT EXISTS LimitNewCapacity ( region TEXT, - period INTEGER - REFERENCES TimePeriod (period), tech_or_group TEXT, + vintage INTEGER + REFERENCES TimePeriod (period), operator TEXT NOT NULL DEFAULT "le" REFERENCES Operator (operator), new_cap REAL, units TEXT, notes TEXT, - PRIMARY KEY (region, period, tech_or_group, operator) + PRIMARY KEY (region, tech_or_group, vintage, operator) ); CREATE TABLE IF NOT EXISTS LimitNewCapacityShare ( region TEXT, - period INTEGER - REFERENCES TimePeriod (period), sub_group TEXT, super_group TEXT, + vintage INTEGER + REFERENCES TimePeriod (period), operator TEXT NOT NULL DEFAULT "le" REFERENCES Operator (operator), share REAL, notes TEXT, - PRIMARY KEY (region, period, sub_group, super_group, operator) + PRIMARY KEY (region, sub_group, super_group, vintage, operator) ); CREATE TABLE IF NOT EXISTS LimitResource ( diff --git a/temoa/db_schema/temoa_schema_v4.sql b/temoa/db_schema/temoa_schema_v4.sql index 2007daa1..7830fd3b 100644 --- a/temoa/db_schema/temoa_schema_v4.sql +++ b/temoa/db_schema/temoa_schema_v4.sql @@ -522,17 +522,16 @@ CREATE TABLE IF NOT EXISTS limit_activity_share CREATE TABLE IF NOT EXISTS limit_annual_capacity_factor ( region TEXT, - period INTEGER + tech_or_group TEXT, + vintage INTEGER REFERENCES time_period (period), - tech TEXT - REFERENCES technology (tech), output_comm TEXT REFERENCES commodity (name), operator TEXT NOT NULL DEFAULT "le" REFERENCES operator (operator), factor REAL, notes TEXT, - PRIMARY KEY (region, period, tech, output_comm, operator), + PRIMARY KEY (region, tech_or_group, vintage, output_comm, operator), CHECK (factor >= 0 AND factor <= 1) ); CREATE TABLE IF NOT EXISTS limit_capacity @@ -564,28 +563,28 @@ CREATE TABLE IF NOT EXISTS limit_capacity_share CREATE TABLE IF NOT EXISTS limit_new_capacity ( region TEXT, - period INTEGER - REFERENCES time_period (period), tech_or_group TEXT, + vintage INTEGER + REFERENCES time_period (period), operator TEXT NOT NULL DEFAULT "le" REFERENCES operator (operator), new_cap REAL, units TEXT, notes TEXT, - PRIMARY KEY (region, period, tech_or_group, operator) + PRIMARY KEY (region, tech_or_group, vintage, operator) ); CREATE TABLE IF NOT EXISTS limit_new_capacity_share ( region TEXT, - period INTEGER - REFERENCES time_period (period), sub_group TEXT, super_group TEXT, + vintage INTEGER + REFERENCES time_period (period), operator TEXT NOT NULL DEFAULT "le" REFERENCES operator (operator), share REAL, notes TEXT, - PRIMARY KEY (region, period, sub_group, super_group, operator) + PRIMARY KEY (region, sub_group, super_group, vintage, operator) ); CREATE TABLE IF NOT EXISTS limit_resource ( @@ -606,13 +605,12 @@ CREATE TABLE IF NOT EXISTS limit_seasonal_capacity_factor REFERENCES time_period (period), season TEXT REFERENCES season_label (season), - tech TEXT - REFERENCES technology (tech), + tech_or_group TEXT, operator TEXT NOT NULL DEFAULT "le" REFERENCES operator (operator), factor REAL, notes TEXT, - PRIMARY KEY(region, period, season, tech, operator) + PRIMARY KEY(region, period, season, tech_or_group, operator) ); CREATE TABLE IF NOT EXISTS limit_tech_input_split ( diff --git a/temoa/model_checking/commodity_network_manager.py b/temoa/model_checking/commodity_network_manager.py index 19b5d839..75afc390 100644 --- a/temoa/model_checking/commodity_network_manager.py +++ b/temoa/model_checking/commodity_network_manager.py @@ -94,7 +94,7 @@ def analyze_network(self) -> bool: orphans_found = any(self.demand_orphans.values()) or any(self.other_orphans.values()) return not orphans_found - def build_filters(self) -> dict[str, ViableSet]: + def build_filters(self, tech_groups: defaultdict[str, set[str]]) -> dict[str, ViableSet]: """ Constructs ViableSet filters based on the valid technologies remaining after the network analysis is complete. @@ -118,6 +118,15 @@ def build_filters(self) -> dict[str, ViableSet]: edge_tuple.output_comm, ) ) + valid_elements['rtvo'].add( + ( + edge_tuple.region, + edge_tuple.tech, + edge_tuple.vintage, + edge_tuple.output_comm, + ) + ) + valid_elements['rpt'].add((edge_tuple.region, p, edge_tuple.tech)) valid_elements['rtv'].add((edge_tuple.region, edge_tuple.tech, edge_tuple.vintage)) valid_elements['rt'].add((edge_tuple.region, edge_tuple.tech)) valid_elements['rpit'].add( @@ -134,8 +143,28 @@ def build_filters(self) -> dict[str, ViableSet]: valid_elements['ic'].add(edge_tuple.input_comm) valid_elements['oc'].add(edge_tuple.output_comm) + for tech_group in tech_groups.get(edge_tuple.tech, {}): + valid_elements['rtvo'].add( + ( + edge_tuple.region, + tech_group, + edge_tuple.vintage, + edge_tuple.output_comm, + ) + ) + valid_elements['rpt'].add((edge_tuple.region, p, tech_group)) + valid_elements['rtv'].add((edge_tuple.region, tech_group, edge_tuple.vintage)) + valid_elements['rt'].add((edge_tuple.region, tech_group)) + valid_elements['rpit'].add( + (edge_tuple.region, p, edge_tuple.input_comm, tech_group) + ) + valid_elements['rpto'].add( + (edge_tuple.region, p, tech_group, edge_tuple.output_comm) + ) + # Good processes that we dont want in the network diagram - for r, _p, t, v in self.orig_data.silent_rptv: + for r, p, t, v in self.orig_data.silent_rptv: + valid_elements['rpt'].add((r, p, t)) valid_elements['rtv'].add((r, t, v)) valid_elements['rt'].add((r, t)) valid_elements['t'].add(t) @@ -147,11 +176,21 @@ def build_filters(self) -> dict[str, ViableSet]: exception_loc=0, exception_vals=ViableSet.REGION_REGEXES, ), + 'rtvo': ViableSet( + elements=valid_elements['rtvo'], + exception_loc=0, + exception_vals=ViableSet.REGION_REGEXES, + ), 'rtv': ViableSet( elements=valid_elements['rtv'], exception_loc=0, exception_vals=ViableSet.REGION_REGEXES, ), + 'rpt': ViableSet( + elements=valid_elements['rpt'], + exception_loc=0, + exception_vals=ViableSet.REGION_REGEXES, + ), 'rt': ViableSet( elements=valid_elements['rt'], exception_loc=0, diff --git a/temoa/tutorial_assets/utopia.sql b/temoa/tutorial_assets/utopia.sql index a9a910f7..0fc2fc03 100644 --- a/temoa/tutorial_assets/utopia.sql +++ b/temoa/tutorial_assets/utopia.sql @@ -802,17 +802,16 @@ CREATE TABLE limit_activity_share CREATE TABLE limit_annual_capacity_factor ( region TEXT, - period INTEGER + tech_or_group TEXT, + vintage INTEGER REFERENCES time_period (period), - tech TEXT - REFERENCES technology (tech), output_comm TEXT REFERENCES commodity (name), operator TEXT NOT NULL DEFAULT "le" REFERENCES operator (operator), factor REAL, notes TEXT, - PRIMARY KEY (region, period, tech, output_comm, operator), + PRIMARY KEY (region, tech_or_group, vintage, output_comm, operator), CHECK (factor >= 0 AND factor <= 1) ); CREATE TABLE limit_capacity @@ -941,28 +940,28 @@ CREATE TABLE limit_growth_new_capacity_delta CREATE TABLE limit_new_capacity ( region TEXT, - period INTEGER - REFERENCES time_period (period), tech_or_group TEXT, + vintage INTEGER + REFERENCES time_period (period), operator TEXT NOT NULL DEFAULT "le" REFERENCES operator (operator), new_cap REAL, units TEXT, notes TEXT, - PRIMARY KEY (region, period, tech_or_group, operator) + PRIMARY KEY (region, tech_or_group, vintage, operator) ); CREATE TABLE limit_new_capacity_share ( region TEXT, - period INTEGER - REFERENCES time_period (period), sub_group TEXT, super_group TEXT, + vintage INTEGER + REFERENCES time_period (period), operator TEXT NOT NULL DEFAULT "le" REFERENCES operator (operator), share REAL, notes TEXT, - PRIMARY KEY (region, period, sub_group, super_group, operator) + PRIMARY KEY (region, sub_group, super_group, vintage, operator) ); CREATE TABLE limit_resource ( @@ -983,13 +982,12 @@ CREATE TABLE limit_seasonal_capacity_factor REFERENCES time_period (period), season TEXT REFERENCES season_label (season), - tech TEXT - REFERENCES technology (tech), + tech_or_group TEXT, operator TEXT NOT NULL DEFAULT "le" REFERENCES operator (operator), factor REAL, notes TEXT, - PRIMARY KEY(region, period, season, tech, operator) + PRIMARY KEY(region, period, season, tech_or_group, operator) ); CREATE TABLE limit_storage_level_fraction ( diff --git a/temoa/utilities/db_migration_v3_to_v3_1.py b/temoa/utilities/db_migration_v3_to_v3_1.py index 63e7b7a8..e5e5180e 100644 --- a/temoa/utilities/db_migration_v3_to_v3_1.py +++ b/temoa/utilities/db_migration_v3_to_v3_1.py @@ -122,6 +122,11 @@ def column_check(old_name: str, new_name: str) -> bool: ("", "TimeSegmentFraction"), ] +period_to_vintage_tables = { + "LimitNewCapacityShare", + "LimitNewCapacity", +} + operator_added_tables = { "EmissionLimit": ("LimitEmission", "le"), "TechOutputSplit": ("LimitTechOutputSplit", "ge"), @@ -135,7 +140,6 @@ def column_check(old_name: str, new_name: str) -> bool: "MinCapacityShare": ("LimitCapacityShare", "ge"), "MinCapacityGroup": ("LimitCapacity", "ge"), "MinCapacity": ("LimitCapacity", "ge"), - "MinAnnualCapacityFactor": ("LimitAnnualCapacityFactor", "ge"), "MinActivityShare": ("LimitActivityShare", "ge"), "MinActivityGroup": ("LimitActivity", "ge"), "MinActivity": ("LimitActivity", "ge"), @@ -146,7 +150,6 @@ def column_check(old_name: str, new_name: str) -> bool: "MaxCapacityShare": ("LimitCapacityShare", "le"), "MaxCapacityGroup": ("LimitCapacity", "le"), "MaxCapacity": ("LimitCapacity", "le"), - "MaxAnnualCapacityFactor": ("LimitAnnualCapacityFactor", "le"), "MaxActivityShare": ("LimitActivityShare", "le"), "MaxActivityGroup": ("LimitActivity", "le"), "MaxActivity": ("LimitActivity", "le"), @@ -156,6 +159,8 @@ def column_check(old_name: str, new_name: str) -> bool: no_transfer = { "MinSeasonalActivity": "LimitSeasonalCapacityFactor", "MaxSeasonalActivity": "LimitSeasonalCapacityFactor", + "MinAnnualCapacityFactor": "LimitAnnualCapacityFactor", + "MaxAnnualCapacityFactor": "LimitAnnualCapacityFactor", "StorageInit": "LimitStorageLevelFraction", } @@ -188,6 +193,22 @@ def column_check(old_name: str, new_name: str) -> bool: ] op_index = new_cols.index("operator") data = [(*row[0:op_index], operator, *row[op_index:len(new_cols)-1]) for row in data] + # if table in period_to_vintage_tables, move period value from period column to vintage column + if new_name in period_to_vintage_tables: + old_cols: list[str] = [ + c[1] for c in con_old.execute(f"PRAGMA table_info({old_name});").fetchall() + ] + period_index = old_cols.index("period") + vintage_index = new_cols.index("vintage") + data = [ + ( + *row[0:period_index], + *row[period_index+1:vintage_index+1], + row[period_index], + *row[vintage_index+1:] + ) + for row in data + ] # construct the query with correct number of placeholders num_placeholders = len(data[0]) @@ -388,6 +409,11 @@ def column_check(old_name: str, new_name: str) -> bool: "manually. ---" ) for old_name, new_name in no_transfer.items(): + # Check if it exists in the old database. If not, no need to warn about it. + try: + con_old.execute(f"SELECT * FROM {old_name}").fetchone() + except sqlite3.OperationalError: + continue print(f"{old_name} to {new_name}") diff --git a/tests/testing_data/mediumville.sql b/tests/testing_data/mediumville.sql index 3cd39f7d..e9b47564 100644 --- a/tests/testing_data/mediumville.sql +++ b/tests/testing_data/mediumville.sql @@ -115,8 +115,8 @@ REPLACE INTO "limit_capacity" VALUES('B',2025,'EH','le',20000.0,'',''); REPLACE INTO "limit_capacity" VALUES('A',2025,'A_tech_grp_1','ge',0.2,'',NULL); REPLACE INTO "limit_capacity" VALUES('A',2025,'A_tech_grp_1','le',6000.0,'',NULL); REPLACE INTO "limit_emission" VALUES('A',2025,'co2','le',10000.0,'gulps',NULL); -REPLACE INTO "limit_new_capacity_share" VALUES('A',2025,'RPS_common','A_tech_grp_1','ge',0.0,''); -REPLACE INTO "limit_new_capacity_share" VALUES('global',2025,'RPS_common','A_tech_grp_1','le',1.0,''); +REPLACE INTO "limit_new_capacity_share" VALUES('A','RPS_common','A_tech_grp_1',2025,'ge',0.0,''); +REPLACE INTO "limit_new_capacity_share" VALUES('global','RPS_common','A_tech_grp_1',2025,'le',1.0,''); REPLACE INTO "limit_resource" VALUES('B','EF','le',9000.0,'clumps',NULL); REPLACE INTO "limit_tech_input_split" VALUES('A',2025,'HYD','EH','ge',0.95,'95% HYD reqt. (other not specified...)'); REPLACE INTO "limit_tech_output_split" VALUES('B',2025,'EH','ELC','ge',0.95,'95% ELC output (there are not others, this is a min)'); diff --git a/tests/testing_data/mediumville_sets.json b/tests/testing_data/mediumville_sets.json index 5c406fed..58deea8a 100644 --- a/tests/testing_data/mediumville_sets.json +++ b/tests/testing_data/mediumville_sets.json @@ -3222,7 +3222,8 @@ ] ], "limit_activity_share_constraint_rpgg": [], - "limit_annual_capacity_factor_constraint_rpto": [], + "limit_annual_capacity_factor_constraint_rptvo": [], + "limit_annual_capacity_factor_constraint_rtvo": [], "limit_capacity_constraint_rpt": [ [ "B", @@ -3276,20 +3277,20 @@ "limit_growth_capacity_constraint_rpt": [], "limit_growth_new_capacity_constraint_rpt": [], "limit_growth_new_capacity_delta_constraint_rpt": [], - "limit_new_capacity_constraint_rpt": [], - "limit_new_capacity_share_constraint_rpgg": [ + "limit_new_capacity_constraint_rtv": [], + "limit_new_capacity_share_constraint_rggv": [ [ "A", - 2025, "RPS_common", "A_tech_grp_1", + 2025, "ge" ], [ "global", - 2025, "RPS_common", "A_tech_grp_1", + 2025, "le" ] ], diff --git a/tests/testing_data/test_system_sets.json b/tests/testing_data/test_system_sets.json index 17bb457e..8c7aa519 100644 --- a/tests/testing_data/test_system_sets.json +++ b/tests/testing_data/test_system_sets.json @@ -39963,7 +39963,8 @@ ] ], "limit_activity_share_constraint_rpgg": [], - "limit_annual_capacity_factor_constraint_rpto": [], + "limit_annual_capacity_factor_constraint_rptvo": [], + "limit_annual_capacity_factor_constraint_rtvo": [], "limit_capacity_constraint_rpt": [], "limit_capacity_share_constraint_rpgg": [], "limit_degrowth_capacity_constraint_rpt": [], @@ -40010,8 +40011,8 @@ "limit_growth_capacity_constraint_rpt": [], "limit_growth_new_capacity_constraint_rpt": [], "limit_growth_new_capacity_delta_constraint_rpt": [], - "limit_new_capacity_constraint_rpt": [], - "limit_new_capacity_share_constraint_rpgg": [], + "limit_new_capacity_constraint_rtv": [], + "limit_new_capacity_share_constraint_rggv": [], "limit_resource_constraint_rt": [], "limit_seasonal_capacity_factor_constraint_rpst": [], "limit_storage_fraction_constraint_rpsdtv": [ diff --git a/tests/testing_data/utopia_sets.json b/tests/testing_data/utopia_sets.json index 77181f16..2b10f202 100644 --- a/tests/testing_data/utopia_sets.json +++ b/tests/testing_data/utopia_sets.json @@ -22105,7 +22105,8 @@ ], "limit_activity_constraint_rpt": [], "limit_activity_share_constraint_rpgg": [], - "limit_annual_capacity_factor_constraint_rpto": [], + "limit_annual_capacity_factor_constraint_rptvo": [], + "limit_annual_capacity_factor_constraint_rtvo": [], "limit_capacity_constraint_rpt": [ [ "utopia", @@ -22182,8 +22183,8 @@ "limit_growth_capacity_constraint_rpt": [], "limit_growth_new_capacity_constraint_rpt": [], "limit_growth_new_capacity_delta_constraint_rpt": [], - "limit_new_capacity_constraint_rpt": [], - "limit_new_capacity_share_constraint_rpgg": [], + "limit_new_capacity_constraint_rtv": [], + "limit_new_capacity_share_constraint_rggv": [], "limit_resource_constraint_rt": [], "limit_seasonal_capacity_factor_constraint_rpst": [], "limit_storage_fraction_constraint_rpsdtv": [],