From 266e2a8e1b373247585f300aa72cb097c79b4eac Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Thu, 24 Apr 2025 10:23:02 -0500 Subject: [PATCH 01/20] Add eta to input damages --- src/dscim/menu/simple_storage.py | 4 ++-- src/dscim/preprocessing/input_damages.py | 19 +++++++++++++++++-- src/dscim/preprocessing/preprocessing.py | 16 +++++++--------- src/dscim/utils/menu_runs.py | 2 ++ 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/dscim/menu/simple_storage.py b/src/dscim/menu/simple_storage.py index 37dd7ce3..9121e2fd 100644 --- a/src/dscim/menu/simple_storage.py +++ b/src/dscim/menu/simple_storage.py @@ -384,8 +384,8 @@ def gdppc(self): def adding_up_damages(self): """This property calls pre-calculated adding-up IR-level 'mean' over batches.""" - mean_cc = f"{self.ce_path}/adding_up_cc.zarr" - mean_no_cc = f"{self.ce_path}/adding_up_no_cc.zarr" + mean_cc = f"{self.ce_path}/adding_up_cc_eta{self.eta}.zarr" + mean_no_cc = f"{self.ce_path}/adding_up_no_cc_eta{self.eta}.zarr" if os.path.exists(mean_cc) and os.path.exists(mean_no_cc): self.logger.info( diff --git a/src/dscim/preprocessing/input_damages.py b/src/dscim/preprocessing/input_damages.py index e55f6bb0..eb43ca1c 100644 --- a/src/dscim/preprocessing/input_damages.py +++ b/src/dscim/preprocessing/input_damages.py @@ -710,6 +710,7 @@ def prep_mortality_damages( outpath, mortality_version, path_econ, + etas, ): ec = EconVars(path_econ=path_econ) @@ -732,6 +733,10 @@ def prep_mortality_damages( scaling_deaths = "epa_row" scaling_costs = "epa_scaled" valuation = "vsl" + elif mortality_version == 9: + scaling_deaths = "epa_popavg" + scaling_costs = "epa_scaled" + valuation = "vly" else: raise ValueError("Mortality version not valid: ", str(mortality_version)) @@ -764,8 +769,14 @@ def prep( valuation=valuation, ).drop(["gcm", "valuation"]) - data = xr.open_mfdataset(paths, preprocess=prep, parallel=True, engine="zarr") - + d_ls = [] + for eta in etas: + paths_ls = [paths.format(i,eta) for i in range(15)] + data = xr.open_mfdataset(paths_ls, preprocess=prep, parallel=True, engine="zarr").assign_coords({'eta': eta}).expand_dims('eta') + d_ls.append(data) + + data = xr.merge(d_ls) + damages = xr.Dataset( { "delta": ( @@ -786,12 +797,14 @@ def prep( damages = damages.chunk( { "batch": 15, + "eta": 1, "ssp": 1, "model": 1, "rcp": 1, "gcm": 1, "year": 10, "region": -1, + "eta": 1, } ) damages.coords.update({"batch": [f"batch{i}" for i in damages.batch.values]}) @@ -808,6 +821,8 @@ def prep( if damages[v].dtype == object: damages[v] = damages[v].astype("unicode") + damages.coords['gcm'] = damages.coords['gcm'].astype('object') + if i == 0: damages.to_zarr( f"{outpath}/impacts-darwin-montecarlo-damages-v{mortality_version}.zarr", diff --git a/src/dscim/preprocessing/preprocessing.py b/src/dscim/preprocessing/preprocessing.py index 8ac0e10f..96714654 100644 --- a/src/dscim/preprocessing/preprocessing.py +++ b/src/dscim/preprocessing/preprocessing.py @@ -43,6 +43,8 @@ def ce_from_chunk( else: raise NotImplementedError("Pass 'cc' or 'no_cc' to reduction.") + calculation = xr.where(calculation > 1e8, np.nan, calculation) + if recipe == "adding_up": result = mean_func( np.maximum( @@ -74,12 +76,6 @@ def reduce_damages( bottom_coding_gdppc=39.39265060424805, zero=False, ): - if recipe == "adding_up": - assert ( - eta is None - ), "Adding up does not take an eta argument. Please set to None." - # client = Client(n_workers=35, memory_limit="9G", threads_per_worker=1) - with open(config) as stream: c = yaml.safe_load(stream) params = c["sectors"][sector] @@ -114,7 +110,7 @@ def reduce_damages( } ce_batch_dims = [i for i in gdppc.dims] + [ - i for i in ds.dims if i not in gdppc.dims and i != "batch" + i for i in ds.dims if i not in gdppc.dims and i != "batch" and i != "eta" ] ce_batch_coords = {c: ds[c].values for c in ce_batch_dims} ce_batch_coords["region"] = [ @@ -129,7 +125,9 @@ def reduce_damages( ).chunk(chunkies) other = xr.open_zarr(damages).chunk(chunkies) - + if "eta" in other.coords: + other = other.sel(eta = eta, drop=True) + out = other.map_blocks( ce_from_chunk, kwargs=dict( @@ -155,7 +153,7 @@ def reduce_damages( if recipe == "adding_up": out.to_zarr( - f"{outpath}/{recipe}_{reduction}.zarr", + f"{outpath}/{recipe}_{reduction}_eta{eta}.zarr", consolidated=True, mode="w", ) diff --git a/src/dscim/utils/menu_runs.py b/src/dscim/utils/menu_runs.py index c8a70845..d1b1429c 100644 --- a/src/dscim/utils/menu_runs.py +++ b/src/dscim/utils/menu_runs.py @@ -104,6 +104,8 @@ def run_ssps( "damage_function_path": f"{conf['paths']['ssp_damage_function_library']}/{sector}/2020/unmasked", "save_files": [ "damage_function_points", + #"uncollapsed_marginal_damages", + #"uncollapsed_discount_factors", "marginal_damages", "discount_factors", "uncollapsed_sccs", From 9bdadc2ee72a8f92afbd3343932fd49c5dc6cfe4 Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Fri, 20 Jun 2025 15:12:05 -0500 Subject: [PATCH 02/20] Format and remove debug --- src/dscim/preprocessing/input_damages.py | 17 +++++++++++------ src/dscim/preprocessing/preprocessing.py | 10 +++++----- src/dscim/utils/menu_runs.py | 2 -- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/dscim/preprocessing/input_damages.py b/src/dscim/preprocessing/input_damages.py index eb43ca1c..46fdb96f 100644 --- a/src/dscim/preprocessing/input_damages.py +++ b/src/dscim/preprocessing/input_damages.py @@ -771,12 +771,18 @@ def prep( d_ls = [] for eta in etas: - paths_ls = [paths.format(i,eta) for i in range(15)] - data = xr.open_mfdataset(paths_ls, preprocess=prep, parallel=True, engine="zarr").assign_coords({'eta': eta}).expand_dims('eta') + paths_ls = [paths.format(i, eta) for i in range(15)] + data = ( + xr.open_mfdataset( + paths_ls, preprocess=prep, parallel=True, engine="zarr" + ) + .assign_coords({"eta": eta}) + .expand_dims("eta") + ) d_ls.append(data) - + data = xr.merge(d_ls) - + damages = xr.Dataset( { "delta": ( @@ -804,7 +810,6 @@ def prep( "gcm": 1, "year": 10, "region": -1, - "eta": 1, } ) damages.coords.update({"batch": [f"batch{i}" for i in damages.batch.values]}) @@ -821,7 +826,7 @@ def prep( if damages[v].dtype == object: damages[v] = damages[v].astype("unicode") - damages.coords['gcm'] = damages.coords['gcm'].astype('object') + damages.coords["gcm"] = damages.coords["gcm"].astype("object") if i == 0: damages.to_zarr( diff --git a/src/dscim/preprocessing/preprocessing.py b/src/dscim/preprocessing/preprocessing.py index 96714654..3c649cdd 100644 --- a/src/dscim/preprocessing/preprocessing.py +++ b/src/dscim/preprocessing/preprocessing.py @@ -43,8 +43,6 @@ def ce_from_chunk( else: raise NotImplementedError("Pass 'cc' or 'no_cc' to reduction.") - calculation = xr.where(calculation > 1e8, np.nan, calculation) - if recipe == "adding_up": result = mean_func( np.maximum( @@ -110,7 +108,9 @@ def reduce_damages( } ce_batch_dims = [i for i in gdppc.dims] + [ - i for i in ds.dims if i not in gdppc.dims and i != "batch" and i != "eta" + i + for i in ds.dims + if i not in gdppc.dims and i != "batch" and i != "eta" ] ce_batch_coords = {c: ds[c].values for c in ce_batch_dims} ce_batch_coords["region"] = [ @@ -126,8 +126,8 @@ def reduce_damages( other = xr.open_zarr(damages).chunk(chunkies) if "eta" in other.coords: - other = other.sel(eta = eta, drop=True) - + other = other.sel(eta=eta, drop=True) + out = other.map_blocks( ce_from_chunk, kwargs=dict( diff --git a/src/dscim/utils/menu_runs.py b/src/dscim/utils/menu_runs.py index d1b1429c..c8a70845 100644 --- a/src/dscim/utils/menu_runs.py +++ b/src/dscim/utils/menu_runs.py @@ -104,8 +104,6 @@ def run_ssps( "damage_function_path": f"{conf['paths']['ssp_damage_function_library']}/{sector}/2020/unmasked", "save_files": [ "damage_function_points", - #"uncollapsed_marginal_damages", - #"uncollapsed_discount_factors", "marginal_damages", "discount_factors", "uncollapsed_sccs", From 5db2861da16a058470e3aa80da316fce47d8c575 Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Fri, 20 Jun 2025 15:29:03 -0500 Subject: [PATCH 03/20] Implement eta in testing --- tests/test_preprocessing.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index 9bdfb47b..a5166cf7 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -69,12 +69,8 @@ def test_subset_USA_reduced_damages(tmp_path, recipe): d = tmp_path / "USA_econ" d.mkdir() - if recipe == "adding_up": - infile = d / f"{sector}/{recipe}_{reduction}.zarr" - outfile = d / f"{sector}_USA/{recipe}_{reduction}.zarr" - else: - infile = d / f"{sector}/{recipe}_{reduction}_eta{eta}.zarr" - outfile = d / f"{sector}_USA/{recipe}_{reduction}_eta{eta}.zarr" + infile = d / f"{sector}/{recipe}_{reduction}_eta{eta}.zarr" + outfile = d / f"{sector}_USA/{recipe}_{reduction}_eta{eta}.zarr" ds_in = xr.Dataset( { @@ -404,14 +400,9 @@ def test_reduce_damages(tmp_path, recipe, eta): + 38.39265060424805 # Since the dummy data gets set to less than the bottom code, set the expected output equal to the bottom code ) - if recipe == "adding_up": - damages_reduced_actual_path = ( - f"{reduced_damages_out}/dummy_sector1/{recipe}_cc.zarr" - ) - else: - damages_reduced_actual_path = ( - f"{reduced_damages_out}/dummy_sector1/{recipe}_cc_eta{eta}.zarr" - ) + damages_reduced_actual_path = ( + f"{reduced_damages_out}/dummy_sector1/{recipe}_cc_eta{eta}.zarr" + ) xr.testing.assert_equal( xr.open_zarr(damages_reduced_actual_path), damages_reduced_out_expected From e39a48c71f0a91617c7b3ba2b47865ff50b2b394 Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Fri, 20 Jun 2025 16:44:54 -0400 Subject: [PATCH 04/20] Change test filename --- .../.zattrs | 0 .../.zgroup | 0 .../.zmetadata | 0 .../cc/.zarray | 0 .../cc/.zattrs | 0 .../cc/0.0.0.0.0.0 | Bin .../cc/0.0.1.0.0.0 | Bin .../cc/0.1.0.0.0.0 | Bin .../cc/0.1.1.0.0.0 | Bin .../cc/1.0.0.0.0.0 | Bin .../cc/1.0.1.0.0.0 | Bin .../cc/1.1.0.0.0.0 | Bin .../cc/1.1.1.0.0.0 | Bin .../gcm/.zarray | 0 .../gcm/.zattrs | 0 .../gcm/0 | Bin .../model/.zarray | 0 .../model/.zattrs | 0 .../model/0 | Bin .../rcp/.zarray | 0 .../rcp/.zattrs | 0 .../rcp/0 | Bin .../region/.zarray | 0 .../region/.zattrs | 0 .../region/0 | Bin .../ssp/.zarray | 0 .../ssp/.zattrs | 0 .../ssp/0 | Bin .../year/.zarray | 0 .../year/.zattrs | 0 .../year/0 | Bin .../.zattrs | 0 .../.zgroup | 0 .../.zmetadata | 0 .../gcm/.zarray | 0 .../gcm/.zattrs | 0 .../gcm/0 | Bin .../model/.zarray | 0 .../model/.zattrs | 0 .../model/0 | Bin .../no_cc/.zarray | 0 .../no_cc/.zattrs | 0 .../no_cc/0.0.0.0.0.0 | Bin .../no_cc/0.0.1.0.0.0 | Bin .../no_cc/0.1.0.0.0.0 | Bin .../no_cc/0.1.1.0.0.0 | Bin .../no_cc/1.0.0.0.0.0 | Bin .../no_cc/1.0.1.0.0.0 | Bin .../no_cc/1.1.0.0.0.0 | Bin .../no_cc/1.1.1.0.0.0 | Bin .../rcp/.zarray | 0 .../rcp/.zattrs | 0 .../rcp/0 | Bin .../region/.zarray | 0 .../region/.zattrs | 0 .../region/0 | Bin .../ssp/.zarray | 0 .../ssp/.zattrs | 0 .../ssp/0 | Bin .../year/.zarray | 0 .../year/.zattrs | 0 .../year/0 | Bin 62 files changed, 0 insertions(+), 0 deletions(-) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/.zattrs (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/.zgroup (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/.zmetadata (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/cc/.zarray (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/cc/.zattrs (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/cc/0.0.0.0.0.0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/cc/0.0.1.0.0.0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/cc/0.1.0.0.0.0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/cc/0.1.1.0.0.0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/cc/1.0.0.0.0.0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/cc/1.0.1.0.0.0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/cc/1.1.0.0.0.0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/cc/1.1.1.0.0.0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/gcm/.zarray (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/gcm/.zattrs (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/gcm/0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/model/.zarray (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/model/.zattrs (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/model/0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/rcp/.zarray (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/rcp/.zattrs (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/rcp/0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/region/.zarray (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/region/.zattrs (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/region/0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/ssp/.zarray (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/ssp/.zattrs (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/ssp/0 (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/year/.zarray (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/year/.zattrs (100%) rename tests/data/CEs/{adding_up_cc.zarr => adding_up_cc_eta1.421158116.zarr}/year/0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/.zattrs (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/.zgroup (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/.zmetadata (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/gcm/.zarray (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/gcm/.zattrs (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/gcm/0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/model/.zarray (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/model/.zattrs (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/model/0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/no_cc/.zarray (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/no_cc/.zattrs (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/no_cc/0.0.0.0.0.0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/no_cc/0.0.1.0.0.0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/no_cc/0.1.0.0.0.0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/no_cc/0.1.1.0.0.0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/no_cc/1.0.0.0.0.0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/no_cc/1.0.1.0.0.0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/no_cc/1.1.0.0.0.0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/no_cc/1.1.1.0.0.0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/rcp/.zarray (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/rcp/.zattrs (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/rcp/0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/region/.zarray (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/region/.zattrs (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/region/0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/ssp/.zarray (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/ssp/.zattrs (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/ssp/0 (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/year/.zarray (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/year/.zattrs (100%) rename tests/data/CEs/{adding_up_no_cc.zarr => adding_up_no_cc_eta1.421158116.zarr}/year/0 (100%) diff --git a/tests/data/CEs/adding_up_cc.zarr/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/.zgroup b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zgroup similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/.zgroup rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zgroup diff --git a/tests/data/CEs/adding_up_cc.zarr/.zmetadata b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zmetadata similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/.zmetadata rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zmetadata diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/0.0.0.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.0.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/0.0.0.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.0.0.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/0.0.1.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.0.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/0.0.1.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.0.1.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/0.1.0.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.1.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/0.1.0.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.1.0.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/0.1.1.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.1.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/0.1.1.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.1.1.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/1.0.0.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.0.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/1.0.0.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.0.0.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/1.0.1.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.0.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/1.0.1.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.0.1.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/1.1.0.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.1.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/1.1.0.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.1.0.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/1.1.1.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.1.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/1.1.1.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.1.1.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/gcm/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/gcm/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/gcm/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/gcm/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/gcm/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/gcm/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/0 diff --git a/tests/data/CEs/adding_up_cc.zarr/model/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/model/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/model/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/model/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/model/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/model/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/0 diff --git a/tests/data/CEs/adding_up_cc.zarr/rcp/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/rcp/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/rcp/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/rcp/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/rcp/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/rcp/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/0 diff --git a/tests/data/CEs/adding_up_cc.zarr/region/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/region/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/region/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/region/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/region/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/region/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/0 diff --git a/tests/data/CEs/adding_up_cc.zarr/ssp/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/ssp/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/ssp/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/ssp/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/ssp/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/ssp/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/0 diff --git a/tests/data/CEs/adding_up_cc.zarr/year/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/year/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/year/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/year/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/year/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/year/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/.zgroup b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zgroup similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/.zgroup rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zgroup diff --git a/tests/data/CEs/adding_up_no_cc.zarr/.zmetadata b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zmetadata similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/.zmetadata rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zmetadata diff --git a/tests/data/CEs/adding_up_no_cc.zarr/gcm/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/gcm/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/gcm/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/gcm/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/gcm/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/gcm/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/model/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/model/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/model/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/model/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/model/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/model/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.0.0.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.0.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.0.0.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.0.0.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.0.1.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.0.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.0.1.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.0.1.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.1.0.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.1.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.1.0.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.1.0.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.1.1.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.1.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.1.1.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.1.1.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.0.0.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.0.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.0.0.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.0.0.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.0.1.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.0.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.0.1.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.0.1.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.1.0.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.1.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.1.0.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.1.0.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.1.1.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.1.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.1.1.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.1.1.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/rcp/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/rcp/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/rcp/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/rcp/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/rcp/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/rcp/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/region/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/region/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/region/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/region/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/region/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/region/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/ssp/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/ssp/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/ssp/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/ssp/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/ssp/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/ssp/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/year/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/year/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/year/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/year/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/year/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/year/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/0 From 02620ef59d550fa0c72fa9ec42ceb0c8611c7497 Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Mon, 23 Jun 2025 13:27:31 -0400 Subject: [PATCH 05/20] Update tests to include mortality etas --- src/dscim/preprocessing/preprocessing.py | 11 +- tests/test_input_damages.py | 160 ++++++++++++----------- tests/test_preprocessing.py | 19 --- tests/test_simple_storage.py | 14 +- 4 files changed, 95 insertions(+), 109 deletions(-) diff --git a/src/dscim/preprocessing/preprocessing.py b/src/dscim/preprocessing/preprocessing.py index 3c649cdd..aa2909d4 100644 --- a/src/dscim/preprocessing/preprocessing.py +++ b/src/dscim/preprocessing/preprocessing.py @@ -270,14 +270,9 @@ def subset_USA_reduced_damages( eta, input_path, ): - if recipe == "adding_up": - ds = xr.open_zarr( - f"{input_path}/{sector}/{recipe}_{reduction}.zarr", - ) - elif recipe == "risk_aversion": - ds = xr.open_zarr( - f"{input_path}/{sector}/{recipe}_{reduction}_eta{eta}.zarr", - ) + ds = xr.open_zarr( + f"{input_path}/{sector}/{recipe}_{reduction}_eta{eta}.zarr", + ) US_territories = [ "USA", diff --git a/tests/test_input_damages.py b/tests/test_input_damages.py index 75d1b0a0..29d3447b 100644 --- a/tests/test_input_damages.py +++ b/tests/test_input_damages.py @@ -1003,80 +1003,82 @@ def test_prep_mortality_damages( Test that prep_mortality_damages correctly reshapes different versions of mortality estimate runs for use in integration system and saves to zarr file """ for b in ["6", "9"]: - ds_in = xr.Dataset( - { - "monetized_costs": ( - [ - "gcm", - "batch", - "ssp", - "rcp", - "model", - "year", - "region", - "scaling", - "valuation", - ], - np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 0), - ), - "monetized_deaths": ( - [ - "gcm", - "batch", - "ssp", - "rcp", - "model", - "year", - "region", - "scaling", - "valuation", - ], - np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 1), - ), - "monetized_histclim_deaths": ( - [ - "gcm", - "batch", - "ssp", - "rcp", - "model", - "year", - "region", - "scaling", - "valuation", - ], - np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 2), - ), - }, - coords={ - "batch": (["batch"], [b]), - "gcm": (["gcm"], ["ACCESS1-0", "GFDL-CM3"]), - "model": (["model"], ["IIASA GDP", "OECD Env-Growth"]), - "rcp": (["rcp"], ["rcp45", "rcp85"]), - "region": (["region"], ["USA.test_region", "ZWE.test_region"]), - "scaling": ( - ["scaling"], - ["epa_scaled", "epa_iso_scaled", "epa_popavg", "epa_row"], - ), - "ssp": (["ssp"], ["SSP2", "SSP3"]), - "valuation": (["valuation"], ["vsl", "vly"]), - "year": (["year"], [2010, 2099]), - }, - ) + for e in ["1.0", "1.34"]: + ds_in = xr.Dataset( + { + "monetized_costs": ( + [ + "gcm", + "batch", + "ssp", + "rcp", + "model", + "year", + "region", + "scaling", + "valuation", + ], + np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 0), + ), + "monetized_deaths": ( + [ + "gcm", + "batch", + "ssp", + "rcp", + "model", + "year", + "region", + "scaling", + "valuation", + ], + np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 1), + ), + "monetized_histclim_deaths": ( + [ + "gcm", + "batch", + "ssp", + "rcp", + "model", + "year", + "region", + "scaling", + "valuation", + ], + np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 2), + ), + }, + coords={ + "batch": (["batch"], [b]), + "gcm": (["gcm"], ["ACCESS1-0", "GFDL-CM3"]), + "model": (["model"], ["IIASA GDP", "OECD Env-Growth"]), + "rcp": (["rcp"], ["rcp45", "rcp85"]), + "region": (["region"], ["USA.test_region", "ZWE.test_region"]), + "scaling": ( + ["scaling"], + ["epa_scaled", "epa_iso_scaled", "epa_popavg", "epa_row"], + ), + "ssp": (["ssp"], ["SSP2", "SSP3"]), + "valuation": (["valuation"], ["vsl", "vly"]), + "year": (["year"], [2010, 2099]), + }, + ) - d = os.path.join(tmp_path, "mortality_in") - if not os.path.exists(d): - os.makedirs(d) - infile = os.path.join(d, f"mortality_damages_batch{b}.zarr") + d = os.path.join(tmp_path, "mortality_in") + if not os.path.exists(d): + os.makedirs(d) + infile = os.path.join(d, f"mortality_damages_batch{b}_eta{e}.zarr") - ds_in.to_zarr(infile) + ds_in.to_zarr(infile) prep_mortality_damages( gcms=["ACCESS1-0", "GFDL-CM3"], - paths=[ - os.path.join(tmp_path, f"mortality_in/mortality_damages_batch{b}.zarr") - for b in [6, 9] - ], + paths=str( + os.path.join( + tmp_path, f"mortality_in/mortality_damages_batch{b}_eta{e}.zarr" + ) + ), vars={ "delta_costs": "monetized_costs", "delta_deaths": "monetized_deaths", @@ -1098,12 +1100,12 @@ def test_prep_mortality_damages( ds_out_expected = xr.Dataset( { "delta": ( - ["gcm", "batch", "ssp", "rcp", "model", "year", "region"], - np.float32(np.full((2, 2, 2, 2, 2, 2, 2), -0.90681089)), + ["gcm", "batch", "ssp", "rcp", "model", "year", "region", "eta"], + np.float32(np.full((2, 2, 2, 2, 2, 2, 2, 2), -0.90681089)), ), "histclim": ( - ["gcm", "batch", "ssp", "rcp", "model", "year", "region"], - np.float32(np.full((2, 2, 2, 2, 2, 2, 2), 2 * 0.90681089)), + ["gcm", "batch", "ssp", "rcp", "model", "year", "region", "eta"], + np.float32(np.full((2, 2, 2, 2, 2, 2, 2, 2), 2 * 0.90681089)), ), }, coords={ @@ -1114,6 +1116,7 @@ def test_prep_mortality_damages( "region": (["region"], ["USA.test_region", "ZWE.test_region"]), "ssp": (["ssp"], ["SSP2", "SSP3"]), "year": (["year"], [2010, 2099]), + "eta": (["eta"], [1.0, 1.34]), }, ) @@ -1127,10 +1130,12 @@ def test_error_prep_mortality_damages(tmp_path): with pytest.raises(ValueError) as excinfo: prep_mortality_damages( gcms=["ACCESS1-0", "GFDL-CM3"], - paths=[ - os.path.join(tmp_path, f"mortality_in/mortality_damages_batch{b}.zarr") - for b in [6, 9] - ], + paths=str( + os.path.join( + tmp_path, + f"mortality_in/mortality_damages_batch{0}_eta{1}.zarr.zarr", + ) + ), vars={ "delta_costs": "monetized_costs", "delta_deaths": "monetized_deaths", @@ -1141,6 +1146,7 @@ def test_error_prep_mortality_damages(tmp_path): path_econ=os.path.join( tmp_path, "econvars_for_test", "econvars_for_test.zarr" ), + etas=[1.0, 1.34], ) assert "Mortality version not valid: " in str(excinfo.value) diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index a5166cf7..2597d5ae 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -253,25 +253,6 @@ def test_sum_AMEL(tmp_path): ) -def test_reduce_damages_error_eta(): - """ - Test that reduce_damages complains when adding up is passed an eta argument - """ - with pytest.raises(AssertionError) as excinfo: - reduce_damages( - "adding_up", - "cc", - 10, - "dummy_sector1", - "/configdirectory/config.yml", - "/reductiondirectory/reduction.zarr", - ) - assert ( - str(excinfo.value) - == "Adding up does not take an eta argument. Please set to None." - ) - - @pytest.mark.parametrize( "recipe, eta", [ diff --git a/tests/test_simple_storage.py b/tests/test_simple_storage.py index b12d2424..5dd4b9f9 100644 --- a/tests/test_simple_storage.py +++ b/tests/test_simple_storage.py @@ -45,8 +45,12 @@ def test_adding_up_damages(stacked_damages): stacked_damages.adding_up_damages, ( ( - xr.open_zarr(f"{stacked_damages.ce_path}/adding_up_no_cc.zarr").no_cc - - xr.open_zarr(f"{stacked_damages.ce_path}/adding_up_cc.zarr").cc + xr.open_zarr( + f"{stacked_damages.ce_path}/adding_up_no_cc_eta1.421158116.zarr" + ).no_cc + - xr.open_zarr( + f"{stacked_damages.ce_path}/adding_up_cc_eta1.421158116.zarr" + ).cc ) * stacked_damages.pop ).sum("region"), @@ -818,8 +822,8 @@ def test_stackeddamages_adding_up_damages(tmp_path): # data sets to a temporary Zarr Store on disk. d = tmp_path / "stackeddamages" d.mkdir() - adding_up_cc_path = d / "adding_up_cc.zarr" - adding_up_no_cc_path = d / "adding_up_no_cc.zarr" + adding_up_cc_path = d / "adding_up_cc_eta1.421158116.zarr" + adding_up_no_cc_path = d / "adding_up_no_cc_eta1.421158116.zarr" # Fake damage data without climate change -- dump to zarr. input_no_cc = xr.Dataset( { @@ -864,7 +868,7 @@ class MockEconVars: save_path="", econ_vars=fake_econvars, climate_vars=fake_climate, - eta=0, + eta=1.421158116, gdppc_bottom_code=0, subset_dict={}, ce_path=str(d), # This MUST be set. From 550f8b46324a7a8cbee9a5e49b9974088c52f84f Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Mon, 23 Jun 2025 13:52:58 -0400 Subject: [PATCH 06/20] Ruff --- src/dscim/preprocessing/preprocessing.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/dscim/preprocessing/preprocessing.py b/src/dscim/preprocessing/preprocessing.py index 2ac15b64..092f2738 100644 --- a/src/dscim/preprocessing/preprocessing.py +++ b/src/dscim/preprocessing/preprocessing.py @@ -82,7 +82,6 @@ def reduce_damages( zero=False, quantreg=False, ): - with open(config) as stream: c = yaml.safe_load(stream) params = c["sectors"][sector] @@ -115,16 +114,14 @@ def reduce_damages( "model": 1, "ssp": 1, } - map_dims = ['eta'] + map_dims = ["eta"] if quantreg: chunkies["batch"] = 1 else: - map_dims.append('batch') + map_dims.append("batch") ce_batch_dims = [i for i in gdppc.dims] + [ - i - for i in ds.dims - if i not in gdppc.dims and i not in map_dims + i for i in ds.dims if i not in gdppc.dims and i not in map_dims ] ce_batch_coords = {c: ds[c].values for c in ce_batch_dims} From 4568d87d7d54ef9e167303676902a8d087d33a32 Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Mon, 23 Jun 2025 13:57:27 -0400 Subject: [PATCH 07/20] Temporarily pin scipy to fix statsmodels import --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7c4cc5bd..47fe2d3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ impactlab-tools==0.6.0 p_tqdm==1.4.2 pyarrow==19.0.1 numcodecs==0.15.1 +scipy==1.15.3 From 884f98d7c80a4559a5c9890e31ad7644657e0781 Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Mon, 23 Jun 2025 14:16:26 -0400 Subject: [PATCH 08/20] Update functions to work with new etas --- src/dscim/preprocessing/preprocessing.py | 17 +++++------------ tests/test_input_damages.py | 1 + 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/dscim/preprocessing/preprocessing.py b/src/dscim/preprocessing/preprocessing.py index 092f2738..92c47c20 100644 --- a/src/dscim/preprocessing/preprocessing.py +++ b/src/dscim/preprocessing/preprocessing.py @@ -310,18 +310,11 @@ def subset_USA_reduced_damages( for var in subset.variables: subset[var].encoding.clear() - if recipe == "adding_up": - subset.to_zarr( - f"{input_path}/{sector}_USA/{recipe}_{reduction}.zarr", - consolidated=True, - mode="w", - ) - elif recipe == "risk_aversion": - subset.to_zarr( - f"{input_path}/{sector}_USA/{recipe}_{reduction}_eta{eta}.zarr", - consolidated=True, - mode="w", - ) + subset.to_zarr( + f"{input_path}/{sector}_USA/{recipe}_{reduction}_eta{eta}.zarr", + consolidated=True, + mode="w", + ) def subset_USA_ssp_econ( diff --git a/tests/test_input_damages.py b/tests/test_input_damages.py index 29d3447b..2d137afe 100644 --- a/tests/test_input_damages.py +++ b/tests/test_input_damages.py @@ -1087,6 +1087,7 @@ def test_prep_mortality_damages( outpath=os.path.join(tmp_path, "mortality_out"), mortality_version=version_test, path_econ=os.path.join(tmp_path, "econvars_for_test", "econvars_for_test.zarr"), + etas=[1.0, 1.34], ) ds_out_actual = xr.open_zarr( From 674aa03ce74daed2efd48ad72d9de12428fdd1ac Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Mon, 23 Jun 2025 14:52:43 -0400 Subject: [PATCH 09/20] Fix test --- tests/test_input_damages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_input_damages.py b/tests/test_input_damages.py index 2d137afe..415cbe0e 100644 --- a/tests/test_input_damages.py +++ b/tests/test_input_damages.py @@ -1002,7 +1002,7 @@ def test_prep_mortality_damages( """ Test that prep_mortality_damages correctly reshapes different versions of mortality estimate runs for use in integration system and saves to zarr file """ - for b in ["6", "9"]: + for b in range(15): for e in ["1.0", "1.34"]: ds_in = xr.Dataset( { From fb6fba1bacc16c5e1f7fcb1dd02d24b24ea828b6 Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Mon, 23 Jun 2025 15:55:40 -0500 Subject: [PATCH 10/20] Test progress --- tests/test_input_damages.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_input_damages.py b/tests/test_input_damages.py index 415cbe0e..8e5d617a 100644 --- a/tests/test_input_damages.py +++ b/tests/test_input_damages.py @@ -993,7 +993,7 @@ def test_calculate_energy_damages( xr.testing.assert_equal(ds_out_expected, ds_out_actual) -@pytest.mark.parametrize("version_test", [0, 1, 4, 5]) +@pytest.mark.parametrize("version_test", [0, 1, 4, 5, 9]) def test_prep_mortality_damages( tmp_path, version_test, @@ -1050,7 +1050,7 @@ def test_prep_mortality_damages( ), }, coords={ - "batch": (["batch"], [b]), + "batch": (["batch"], [f"{b}"]), "gcm": (["gcm"], ["ACCESS1-0", "GFDL-CM3"]), "model": (["model"], ["IIASA GDP", "OECD Env-Growth"]), "rcp": (["rcp"], ["rcp45", "rcp85"]), @@ -1063,7 +1063,7 @@ def test_prep_mortality_damages( "valuation": (["valuation"], ["vsl", "vly"]), "year": (["year"], [2010, 2099]), }, - ) + ).expand_dims('mortality') d = os.path.join(tmp_path, "mortality_in") if not os.path.exists(d): @@ -1076,7 +1076,7 @@ def test_prep_mortality_damages( gcms=["ACCESS1-0", "GFDL-CM3"], paths=str( os.path.join( - tmp_path, f"mortality_in/mortality_damages_batch{b}_eta{e}.zarr" + tmp_path, "mortality_in/mortality_damages_batch{0}_eta{1}.zarr" ) ), vars={ @@ -1102,15 +1102,15 @@ def test_prep_mortality_damages( { "delta": ( ["gcm", "batch", "ssp", "rcp", "model", "year", "region", "eta"], - np.float32(np.full((2, 2, 2, 2, 2, 2, 2, 2), -0.90681089)), + np.float32(np.full((2, 15, 2, 2, 2, 2, 2, 2), -0.90681089)), ), "histclim": ( ["gcm", "batch", "ssp", "rcp", "model", "year", "region", "eta"], - np.float32(np.full((2, 2, 2, 2, 2, 2, 2, 2), 2 * 0.90681089)), + np.float32(np.full((2, 15, 2, 2, 2, 2, 2, 2), 2 * 0.90681089)), ), }, coords={ - "batch": (["batch"], ["batch6", "batch9"]), + "batch": (["batch"], ["batch" + str(i) for i in [0, 1, 10, 11, 12, 13, 14, 2, 3, 4, 5, 6, 7, 8, 9,]]), "gcm": (["gcm"], ["ACCESS1-0", "GFDL-CM3"]), "model": (["model"], ["IIASA GDP", "OECD Env-Growth"]), "rcp": (["rcp"], ["rcp45", "rcp85"]), From 3a0f2e6c513ee6bbf4a2fb7fd038b426ba608a49 Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Tue, 24 Jun 2025 11:35:02 -0400 Subject: [PATCH 11/20] Fix tests --- tests/test_input_damages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_input_damages.py b/tests/test_input_damages.py index 8e5d617a..b16e3e0a 100644 --- a/tests/test_input_damages.py +++ b/tests/test_input_damages.py @@ -1110,7 +1110,7 @@ def test_prep_mortality_damages( ), }, coords={ - "batch": (["batch"], ["batch" + str(i) for i in [0, 1, 10, 11, 12, 13, 14, 2, 3, 4, 5, 6, 7, 8, 9,]]), + "batch": (["batch"], ["batch" + str(i) for i in range(15)]), "gcm": (["gcm"], ["ACCESS1-0", "GFDL-CM3"]), "model": (["model"], ["IIASA GDP", "OECD Env-Growth"]), "rcp": (["rcp"], ["rcp45", "rcp85"]), @@ -1121,7 +1121,7 @@ def test_prep_mortality_damages( }, ) - xr.testing.assert_equal(ds_out_expected, ds_out_actual) + xr.testing.assert_equal(ds_out_expected, ds_out_actual.sel(ds_out_expected.coords)) def test_error_prep_mortality_damages(tmp_path): From 664572dcf179de4a35a731fdf68d999c645b768a Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Tue, 24 Jun 2025 11:47:01 -0400 Subject: [PATCH 12/20] Fix tests --- tests/test_input_damages.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_input_damages.py b/tests/test_input_damages.py index b16e3e0a..65cbbe44 100644 --- a/tests/test_input_damages.py +++ b/tests/test_input_damages.py @@ -1063,7 +1063,7 @@ def test_prep_mortality_damages( "valuation": (["valuation"], ["vsl", "vly"]), "year": (["year"], [2010, 2099]), }, - ).expand_dims('mortality') + ).expand_dims("mortality") d = os.path.join(tmp_path, "mortality_in") if not os.path.exists(d): @@ -1121,7 +1121,10 @@ def test_prep_mortality_damages( }, ) - xr.testing.assert_equal(ds_out_expected, ds_out_actual.sel(ds_out_expected.coords)) + xr.testing.assert_equal( + ds_out_expected, + ds_out_actual.sel(ds_out_expected.coords).transpose(ds_out_expected.dims), + ) def test_error_prep_mortality_damages(tmp_path): From e26d4534ea6994171c61d8c8ed2021f274650fd6 Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Tue, 24 Jun 2025 12:00:59 -0400 Subject: [PATCH 13/20] Fix tests --- tests/test_input_damages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_input_damages.py b/tests/test_input_damages.py index 65cbbe44..33c4f60e 100644 --- a/tests/test_input_damages.py +++ b/tests/test_input_damages.py @@ -1123,7 +1123,7 @@ def test_prep_mortality_damages( xr.testing.assert_equal( ds_out_expected, - ds_out_actual.sel(ds_out_expected.coords).transpose(ds_out_expected.dims), + ds_out_actual.sel(ds_out_expected.coords).transpose(*list(ds_out_expected.dims)), ) From c6516be57aac097f419e300fb5798f2638d5dd6c Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Tue, 24 Jun 2025 12:14:06 -0400 Subject: [PATCH 14/20] Fix tests --- tests/test_input_damages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_input_damages.py b/tests/test_input_damages.py index 33c4f60e..cc804057 100644 --- a/tests/test_input_damages.py +++ b/tests/test_input_damages.py @@ -1063,7 +1063,7 @@ def test_prep_mortality_damages( "valuation": (["valuation"], ["vsl", "vly"]), "year": (["year"], [2010, 2099]), }, - ).expand_dims("mortality") + ) d = os.path.join(tmp_path, "mortality_in") if not os.path.exists(d): @@ -1123,7 +1123,7 @@ def test_prep_mortality_damages( xr.testing.assert_equal( ds_out_expected, - ds_out_actual.sel(ds_out_expected.coords).transpose(*list(ds_out_expected.dims)), + ds_out_actual, ) From 1577e16eaab6717ae4158689b3dbaf13a0a7e458 Mon Sep 17 00:00:00 2001 From: Jonah Gilbert Date: Tue, 24 Jun 2025 12:32:04 -0400 Subject: [PATCH 15/20] Fix tests --- tests/test_input_damages.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_input_damages.py b/tests/test_input_damages.py index cc804057..48ddbd9e 100644 --- a/tests/test_input_damages.py +++ b/tests/test_input_damages.py @@ -1123,7 +1123,9 @@ def test_prep_mortality_damages( xr.testing.assert_equal( ds_out_expected, - ds_out_actual, + ds_out_actual.sel(ds_out_expected.coords).transpose( + *list(ds_out_expected.dims) + ), ) From 64dcb104ea1cac5823fb619d5535bf88fb0cd227 Mon Sep 17 00:00:00 2001 From: Sebastian Cadavid Sanchez Date: Fri, 5 Dec 2025 16:45:02 -0600 Subject: [PATCH 16/20] Update CHANGELOG for reduce_etas branch (PR #417) --- CHANGELOG.md | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fc980a4..de394e74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,15 +4,45 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] + +## Unreleased + +### Changed + +- Standardized eta handling across all damage reduction recipes. Previously, the `adding_up` recipe did not use eta parameters while `risk_aversion` did. Now all recipes require and use eta values consistently. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) + - File naming convention updated to include eta value for all reduced damage outputs (e.g., `adding_up_cc_eta1.0.zarr`) + - Simplified `reduce_damages()` and `subset_USA_reduced_damages()` functions by removing conditional logic for eta-less processing + - Updated `StackedDamages.adding_up_damages` property to reference eta-specific file paths +- Refactored mortality damage preprocessing to support eta-aware damage loading. The `prep_mortality_damages()` function now accepts multiple eta values and merges damage data across the eta dimension. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) +- Added support for mortality version 9 with VLY (Value of a Life Year) valuation using EPA population-averaged scaling. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) +- Updated chunking strategy in `reduce_damages()` to handle eta as a mapped dimension alongside batch operations, improving consistency in batch dimension handling for both standard and quantile regression workflows. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) + +### Fixed + +- Pinned `scipy=1.15.3` to resolve `statsmodels==0.14.4` import issues. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) +- Updated all unit tests to work with eta-aware damage processing, including test fixtures for mortality damages and adding-up calculations. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) + +### Removed + +- Removed assertion preventing `adding_up` recipe from accepting eta arguments, as this recipe now processes damages with eta values like all other recipes. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) + +## [0.7.0] - 2025-08-15 + +### Added +- [Documentation pages](climateimpactlab.github.io/dscim/) added using mkdocs ([PR #254](https://github.com/ClimateImpactLab/dscim/pull/254), [@JMGilbert](https://github.com/JMGilbert)) +- Added discounting option `constant_gwr`, which applies discounting across SSPs ([PR #405](https://github.com/ClimateImpactLab/dscim/pull/405), [@JMGilbert](https://github.com/JMGilbert)). ### Changed +- The function signature for `calculate_labor_batch_damages()` in `src/dscim/preprocessing/input_damages.py` was updated to include additional args with default values that allow the labor SCC application to run without modifying `dscim` code in the future. This is backwards compatible. ([PR #415](https://github.com/ClimateImpactLab/dscim/pull/415), [@JMGilbert](https://github.com/JMGilbert)). - Python version for running automated tests in CI upgraded from Python 3.10 to 3.12 ([PR #270](https://github.com/ClimateImpactLab/dscim/pull/270), [@brews](https://github.com/brews)). ### Fixed +- Fixed how quantile regression SCCs (`quantreg`) are calculated by allowing for the full cloud of damage points in the damage function fit stage (previously the `batch` dimension was incorrectly reduced before damage function fit even if `quantreg=True`) ([PR #405](https://github.com/ClimateImpactLab/dscim/pull/405), [@JMGilbert](https://github.com/JMGilbert)). - Minor code cleanup. Switch old %-string formatting to use f-strings ([PR #351](https://github.com/ClimateImpactLab/dscim/pull/351), [@brews](https://github.com/brews)). +- Pin `numcodecs` package to 0.15.1 to fix automated tests in CI. This works with `zarr < 3`. ([PR #406](https://github.com/ClimateImpactLab/dscim/pull/406), [@JMGilbert](https://github.com/JMGilbert)). +- Pin `statsmodels` to 0.14.5 to fix automated tests in CI. ([PR #429](https://github.com/ClimateImpactLab/dscim/pull/429), [@C1587S](https://github.com/C1587S)). ### Removed @@ -137,7 +167,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Initial release. -[unreleased]: https://github.com/climateimpactlab/dscim/compare/v0.6.0...HEAD +[unreleased]: https://github.com/climateimpactlab/dscim/compare/v0.7.0...HEAD +[0.7.0]: https://github.com/climateimpactlab/dscim/compare/v0.6.0...v0.7.0 [0.6.0]: https://github.com/climateimpactlab/dscim/compare/v0.5.0...v0.6.0 [0.5.0]: https://github.com/climateimpactlab/dscim/compare/v0.4.0...v0.5.0 [0.4.0]: https://github.com/climateimpactlab/dscim/compare/v0.3.0...v0.4.0 From 6852992df4aea0c5caac5bd5cde6c14a1539cbd0 Mon Sep 17 00:00:00 2001 From: Sebastian Cadavid Sanchez Date: Fri, 16 Jan 2026 10:54:34 -0600 Subject: [PATCH 17/20] Add geography aggregation methods and damages_dataset to MainRecipe --- src/dscim/menu/main_recipe.py | 175 ++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/src/dscim/menu/main_recipe.py b/src/dscim/menu/main_recipe.py index 483c431f..96d36af7 100644 --- a/src/dscim/menu/main_recipe.py +++ b/src/dscim/menu/main_recipe.py @@ -107,6 +107,9 @@ def __init__( extrap_formula=None, fair_dims=None, save_files=None, + geography="globe", + country_mapping_path=None, + individual_region=None, **kwargs, ): if scc_quantiles is None: @@ -232,6 +235,12 @@ def __init__( self.extrap_formula = extrap_formula self.fair_dims = fair_dims self.save_files = save_files + self.geography = geography + self.individual_region = individual_region + if country_mapping_path is not None: + self.country_mapping = pd.read_csv(country_mapping_path) + else: + self.country_mapping = None self.__dict__.update(**kwargs) self.kwargs = kwargs @@ -456,6 +465,142 @@ def collapsed_pop(self): pop = self.pop.mean(["model", "ssp"]) return pop + def _aggregate_by_geography( + self, + data: xr.DataArray, + geography: str, + ) -> xr.DataArray: + """Aggregate data by geography type. + + Parameters + ---------- + data : xr.DataArray + Data with 'region' dimension to aggregate + geography : str + One of "ir" (impact region), "country", or "globe" + + Returns + ------- + xr.DataArray + Aggregated data according to geography + """ + if geography == "ir": + return data # No aggregation needed + + elif geography == "country": + if ( + hasattr(self, "country_mapping") + and self.country_mapping is not None + ): + return self._aggregate_to_country(data) + else: + raise ValueError( + "Country aggregation requires country_mapping parameter" + ) + + elif geography in ("globe", "global"): + return ( + data.sum(dim="region") + .assign_coords({"region": "globe"}) + .expand_dims("region") + ) + + else: + raise ValueError( + f"Unknown geography: {geography}. " + f"Valid options: 'ir', 'country', 'globe'" + ) + + def _aggregate_to_country(self, data: xr.DataArray) -> xr.DataArray: + """Aggregate to country level using mapping. + + Parameters + ---------- + data : xr.DataArray + Data with 'region' dimension to aggregate + + Returns + ------- + xr.DataArray + Data aggregated to country level + """ + mapping = { + row["ISO"]: row["MatchedISO"] + if str(row["MatchedISO"]) != "nan" + else "nopop" + for _, row in self.country_mapping.iterrows() + } + territories = [ + mapping.get(str(r)[:3], "unknown") for r in data.region.values + ] + return data.assign_coords({"region": territories}).groupby("region").sum() + + def damages_dataset(self, geography: str = "globe") -> xr.Dataset: + """Calculate damages as xarray Dataset. + + Parameters + ---------- + geography : str, optional + One of "ir" (impact region), "country", or "globe" (default) + + Returns + ------- + xr.Dataset + Damages dataset with regional aggregation + """ + # Get raw calculated damages (xr.DataArray) + damages = self.calculated_damages + pop = self.collapsed_pop + + # Apply geography aggregation + aggregated = self._aggregate_by_geography(damages * pop, geography) + + ds = aggregated.to_dataset(name="damages") + + # Add metadata for GWR discounting + if "gwr" in self.discounting_type: + ds = ds.assign_coords( + ssp=str(list(self.gdp.ssp.values)), + model=str(list(self.gdp.model.values)), + ) + + return ds + + def _filter_illegal_combinations_xr(self, ds: xr.Dataset) -> xr.Dataset: + """Filter illegal SSP/RCP combinations (xarray version). + + Parameters + ---------- + ds : xr.Dataset + Dataset to filter + + Returns + ------- + xr.Dataset + Filtered dataset with illegal combinations set to NaN + """ + if "ssp" in ds.coords and any( + ssp in ds.ssp.values for ssp in ["SSP1", "SSP5"] + ): + self.logger.info("Dropping illegal model combinations.") + illegal = ((ds.ssp == "SSP1") & (ds.rcp == "rcp85")) | ( + (ds.ssp == "SSP5") & (ds.rcp == "rcp45") + ) + for var in ["anomaly", "gmsl"]: + if var in ds: + ds[var] = xr.where(illegal, np.nan, ds[var]) + + if "agriculture" in self.sector: + self.logger.info("Dropping illegal model combinations for agriculture.") + if "anomaly" in ds: + ds["anomaly"] = xr.where( + (ds.gcm == "ACCESS1-0") & (ds.rcp == "rcp85"), + np.nan, + ds["anomaly"], + ) + + return ds + @abstractmethod def ce_cc_calculation(self): """Calculate CE damages depending on discount type""" @@ -491,6 +636,13 @@ def damage_function_points(self) -> pd.DataFrame: -------- pd.DataFrame """ + if self.geography != "globe": + return self._damage_function_points_xarray() + else: + return self._damage_function_points_pandas() + + def _damage_function_points_pandas(self) -> pd.DataFrame: + """Original pandas-based implementation for backward compatibility.""" df = self.global_damages_calculation() if "slr" in df.columns: @@ -515,6 +667,29 @@ def damage_function_points(self) -> pd.DataFrame: return df + def _damage_function_points_xarray(self) -> pd.DataFrame: + """xarray-based implementation for geography support.""" + ds = self.damages_dataset(geography=self.geography) + + # Merge climate data using xarray + climate_vars = [] + if "slr" in ds.coords: + gmsl = self.climate.gmsl.set_index(["slr", "year"]).to_xarray() + climate_vars.append(gmsl) + if "gcm" in ds.coords: + gmst = self.climate.gmst.set_index(["gcm", "rcp", "year"]).to_xarray() + climate_vars.append(gmst) + + if climate_vars: + climate_ds = xr.merge(climate_vars) + ds = xr.merge([ds, climate_ds]).sel(year=ds.year) + + # Filter illegal combinations + ds = self._filter_illegal_combinations_xr(ds) + + # Convert to DataFrame at the boundary + return ds.to_dataframe().reset_index() + def damage_function_calculation(self, damage_function_points, global_consumption): """The damage function model fit may be : (1) ssp specific, (2) ssp-model specific, (3) unique across ssp-model. This depends on the type of discounting. In each case the input data passed to the fitting functions and the formatting of the returned From 6ed821fb917c2e332458bb5a19be702fa7f6035e Mon Sep 17 00:00:00 2001 From: Sebastian Cadavid Sanchez Date: Fri, 16 Jan 2026 10:55:20 -0600 Subject: [PATCH 18/20] Add geography tests and menu_instance_with_geography fixture --- tests/conftest.py | 46 ++++++++++ tests/test_geography.py | 181 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 tests/test_geography.py diff --git a/tests/conftest.py b/tests/conftest.py index f081d348..45b3149f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -193,3 +193,49 @@ def save_ssprff_econ(tmp_path): ssp_econ.to_zarr(d / "integration-econ-bc39.zarr") rff_econ.to_netcdf(d / "rff_global_socioeconomics.nc4") + + +@pytest.fixture(scope="module") +def menu_instance_with_geography(menu_class, discount_types, econ, climate): + """Menu instance with geography parameter.""" + datadir = os.path.join(os.path.dirname(__file__), "data") + yield menu_class( + sector_path=[{"dummy_sector": os.path.join(datadir, "damages")}], + save_path=None, + discrete_discounting=True, + econ_vars=econ, + climate_vars=climate, + fit_type="ols", + variable=[{"dummy_sector": "damages"}], + sector="dummy_sector", + discounting_type=discount_types, + ext_method="global_c_ratio", + save_files=[ + "damage_function_points", + "global_consumption", + "damage_function_coefficients", + "damage_function_fit", + ], + ce_path=os.path.join(datadir, "CEs"), + subset_dict={ + "ssp": ["SSP2", "SSP3", "SSP4"], + "region": [ + "IND.21.317.1249", + "CAN.2.33.913", + "USA.14.608", + "EGY.11", + "SDN.4.11.50.164", + "NGA.25.510", + "SAU.7", + "RUS.16.430.430", + "SOM.2.5", + ], + }, + formula="damages ~ -1 + anomaly + np.power(anomaly, 2)", + extrap_formula=None, + fair_aggregation=["median_params", "ce", "mean"], + weitzman_parameter=[0.1], + geography="globe", + country_mapping_path=None, + individual_region=None, + ) diff --git a/tests/test_geography.py b/tests/test_geography.py new file mode 100644 index 00000000..d6fc4d07 --- /dev/null +++ b/tests/test_geography.py @@ -0,0 +1,181 @@ +"""Tests for geography functionality and backward compatibility.""" + +import pandas +import xarray as xr +import numpy as np +import pytest + +from dscim.menu.risk_aversion import RiskAversionRecipe + + +class TestGlobeGeographyEquivalence: + """Tests that xarray path produces same results as pandas path for globe.""" + + @pytest.mark.parametrize("menu_class", [RiskAversionRecipe], indirect=True) + @pytest.mark.parametrize("discount_types", ["euler_ramsey"], indirect=True) + def test_damages_dataset_equals_global_damages_calculation(self, menu_instance): + df_pandas = menu_instance.global_damages_calculation() + ds_xarray = menu_instance.damages_dataset(geography="globe") + + df_xarray = ds_xarray.to_dataframe().reset_index() + + assert "damages" in df_pandas.columns + assert "damages" in df_xarray.columns + + damages_pandas = df_pandas["damages"].sort_values().reset_index(drop=True) + damages_xarray = df_xarray["damages"].sort_values().reset_index(drop=True) + + np.testing.assert_allclose( + damages_pandas.values, + damages_xarray.values, + rtol=1e-10, + atol=1e-10, + ) + + @pytest.mark.parametrize("menu_class", [RiskAversionRecipe], indirect=True) + @pytest.mark.parametrize( + "discount_types", ["euler_ramsey", "euler_gwr", "constant"], indirect=True + ) + def test_damages_dataset_returns_dataset(self, menu_instance): + result = menu_instance.damages_dataset(geography="globe") + assert isinstance(result, xr.Dataset) + assert "damages" in result.data_vars + + +class TestGeographyAggregation: + """Tests for _aggregate_by_geography method.""" + + @pytest.mark.parametrize("menu_class", [RiskAversionRecipe], indirect=True) + @pytest.mark.parametrize("discount_types", ["euler_ramsey"], indirect=True) + def test_aggregate_globe_sums_all_regions(self, menu_instance): + damages = menu_instance.calculated_damages * menu_instance.collapsed_pop + + expected = damages.sum(dim="region") + actual = menu_instance._aggregate_by_geography(damages, "globe") + + assert actual.region.values == ["globe"] + + actual_values = actual.squeeze(dim="region", drop=True) + xr.testing.assert_allclose(expected, actual_values) + + @pytest.mark.parametrize("menu_class", [RiskAversionRecipe], indirect=True) + @pytest.mark.parametrize("discount_types", ["euler_ramsey"], indirect=True) + def test_aggregate_ir_preserves_regions(self, menu_instance): + damages = menu_instance.calculated_damages * menu_instance.collapsed_pop + + result = menu_instance._aggregate_by_geography(damages, "ir") + + xr.testing.assert_allclose(result, damages) + + @pytest.mark.parametrize("menu_class", [RiskAversionRecipe], indirect=True) + @pytest.mark.parametrize("discount_types", ["euler_ramsey"], indirect=True) + def test_invalid_geography_raises_error(self, menu_instance): + damages = menu_instance.calculated_damages * menu_instance.collapsed_pop + + with pytest.raises(ValueError, match="Unknown geography"): + menu_instance._aggregate_by_geography(damages, "invalid_geography") + + @pytest.mark.parametrize("menu_class", [RiskAversionRecipe], indirect=True) + @pytest.mark.parametrize("discount_types", ["euler_ramsey"], indirect=True) + def test_country_without_mapping_raises_error(self, menu_instance): + damages = menu_instance.calculated_damages * menu_instance.collapsed_pop + + menu_instance.country_mapping = None + + with pytest.raises(ValueError, match="country_mapping"): + menu_instance._aggregate_by_geography(damages, "country") + + +class TestBackwardCompatibility: + """Tests for backward compatibility with existing API.""" + + @pytest.mark.parametrize("menu_class", [RiskAversionRecipe], indirect=True) + @pytest.mark.parametrize("discount_types", ["euler_ramsey"], indirect=True) + def test_global_damages_calculation_returns_dataframe(self, menu_instance): + result = menu_instance.global_damages_calculation() + assert isinstance(result, pandas.DataFrame) + assert "region" not in result.columns + + @pytest.mark.parametrize("menu_class", [RiskAversionRecipe], indirect=True) + @pytest.mark.parametrize("discount_types", ["euler_ramsey"], indirect=True) + def test_damage_function_points_returns_dataframe(self, menu_instance): + result = menu_instance.damage_function_points + assert isinstance(result, pandas.DataFrame) + + @pytest.mark.parametrize("menu_class", [RiskAversionRecipe], indirect=True) + @pytest.mark.parametrize("discount_types", ["euler_ramsey"], indirect=True) + def test_default_geography_is_globe(self, menu_instance): + assert menu_instance.geography == "globe" + + +class TestDualPathEquivalence: + """Tests for pandas vs xarray path equivalence.""" + + @pytest.mark.parametrize("menu_class", [RiskAversionRecipe], indirect=True) + @pytest.mark.parametrize("discount_types", ["euler_ramsey"], indirect=True) + def test_pandas_path_used_for_globe(self, menu_instance): + assert menu_instance.geography == "globe" + + result = menu_instance.damage_function_points + assert isinstance(result, pandas.DataFrame) + + expected = menu_instance._damage_function_points_pandas() + pandas.testing.assert_frame_equal(result, expected) + + @pytest.mark.parametrize("menu_class", [RiskAversionRecipe], indirect=True) + @pytest.mark.parametrize("discount_types", ["euler_ramsey"], indirect=True) + def test_xarray_path_matches_pandas_path_for_globe(self, menu_instance): + # Compare full pipeline: damages, climate merge, illegal filtering + pandas_result = menu_instance._damage_function_points_pandas() + + original_geography = menu_instance.geography + menu_instance.geography = "globe" + xarray_result = menu_instance._damage_function_points_xarray() + menu_instance.geography = original_geography + + assert isinstance(pandas_result, pandas.DataFrame) + assert isinstance(xarray_result, pandas.DataFrame) + + assert "damages" in pandas_result.columns + assert "damages" in xarray_result.columns + + sort_cols = [c for c in ["year", "ssp", "model", "gcm", "rcp"] if c in pandas_result.columns] + pandas_sorted = pandas_result.sort_values(sort_cols).reset_index(drop=True) + xarray_sorted = xarray_result.sort_values(sort_cols).reset_index(drop=True) + + np.testing.assert_allclose( + pandas_sorted["damages"].values, + xarray_sorted["damages"].values, + rtol=1e-10, + atol=1e-10, + ) + + if "anomaly" in pandas_sorted.columns and "anomaly" in xarray_sorted.columns: + pandas_nan = pandas_sorted["anomaly"].isna() + xarray_nan = xarray_sorted["anomaly"].isna() + assert (pandas_nan == xarray_nan).all() + + pandas_valid = pandas_sorted.loc[~pandas_nan, "anomaly"].values + xarray_valid = xarray_sorted.loc[~xarray_nan, "anomaly"].values + np.testing.assert_allclose( + pandas_valid, + xarray_valid, + rtol=1e-10, + atol=1e-10, + ) + + +class TestCountryAggregation: + """Tests for country-level aggregation.""" + + @pytest.mark.skip(reason="Requires country_mapping fixture") + def test_country_aggregation(self): + pass + + +class TestIndividualRegion: + """Tests for individual region calculations.""" + + @pytest.mark.skip(reason="For future individual_region support") + def test_individual_region_filter(self): + pass From 07c08c498fe3941aeb55fdd31a64d4ee259cf43e Mon Sep 17 00:00:00 2001 From: Sebastian Cadavid Sanchez Date: Fri, 16 Jan 2026 12:22:39 -0600 Subject: [PATCH 19/20] Fix ruff formatting --- src/dscim/menu/main_recipe.py | 31 ++++++++++++------------------- tests/test_geography.py | 6 +++++- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/dscim/menu/main_recipe.py b/src/dscim/menu/main_recipe.py index 96d36af7..eeadfdab 100644 --- a/src/dscim/menu/main_recipe.py +++ b/src/dscim/menu/main_recipe.py @@ -247,17 +247,17 @@ def __init__( self.logger = logging.getLogger(__name__) if self.quantreg_quantiles is not None: - assert len(self.quantreg_quantiles) == len( - self.quantreg_weights - ), "Length of quantreg quantiles does not match length of weights." + assert len(self.quantreg_quantiles) == len(self.quantreg_weights), ( + "Length of quantreg quantiles does not match length of weights." + ) - assert ( - self.discounting_type in self.DISCOUNT_TYPES - ), f"Discount type not implemented. Try one of {self.DISCOUNT_TYPES}." + assert self.discounting_type in self.DISCOUNT_TYPES, ( + f"Discount type not implemented. Try one of {self.DISCOUNT_TYPES}." + ) - assert ( - self.formula in self.FORMULAS - ), f"Formula not implemented. Try one of {self.FORMULAS}." + assert self.formula in self.FORMULAS, ( + f"Formula not implemented. Try one of {self.FORMULAS}." + ) # Set stream of discounts to None if discounting_type is 'constant' # 'constant_model_collapsed' should be here except that we allow @@ -488,10 +488,7 @@ def _aggregate_by_geography( return data # No aggregation needed elif geography == "country": - if ( - hasattr(self, "country_mapping") - and self.country_mapping is not None - ): + if hasattr(self, "country_mapping") and self.country_mapping is not None: return self._aggregate_to_country(data) else: raise ValueError( @@ -530,9 +527,7 @@ def _aggregate_to_country(self, data: xr.DataArray) -> xr.DataArray: else "nopop" for _, row in self.country_mapping.iterrows() } - territories = [ - mapping.get(str(r)[:3], "unknown") for r in data.region.values - ] + territories = [mapping.get(str(r)[:3], "unknown") for r in data.region.values] return data.assign_coords({"region": territories}).groupby("region").sum() def damages_dataset(self, geography: str = "globe") -> xr.Dataset: @@ -579,9 +574,7 @@ def _filter_illegal_combinations_xr(self, ds: xr.Dataset) -> xr.Dataset: xr.Dataset Filtered dataset with illegal combinations set to NaN """ - if "ssp" in ds.coords and any( - ssp in ds.ssp.values for ssp in ["SSP1", "SSP5"] - ): + if "ssp" in ds.coords and any(ssp in ds.ssp.values for ssp in ["SSP1", "SSP5"]): self.logger.info("Dropping illegal model combinations.") illegal = ((ds.ssp == "SSP1") & (ds.rcp == "rcp85")) | ( (ds.ssp == "SSP5") & (ds.rcp == "rcp45") diff --git a/tests/test_geography.py b/tests/test_geography.py index d6fc4d07..028225d5 100644 --- a/tests/test_geography.py +++ b/tests/test_geography.py @@ -139,7 +139,11 @@ def test_xarray_path_matches_pandas_path_for_globe(self, menu_instance): assert "damages" in pandas_result.columns assert "damages" in xarray_result.columns - sort_cols = [c for c in ["year", "ssp", "model", "gcm", "rcp"] if c in pandas_result.columns] + sort_cols = [ + c + for c in ["year", "ssp", "model", "gcm", "rcp"] + if c in pandas_result.columns + ] pandas_sorted = pandas_result.sort_values(sort_cols).reset_index(drop=True) xarray_sorted = xarray_result.sort_values(sort_cols).reset_index(drop=True) From 31010aa3da5886281ddaa79cbb62f3741fa54738 Mon Sep 17 00:00:00 2001 From: Sebastian Cadavid Sanchez Date: Fri, 16 Jan 2026 12:34:31 -0600 Subject: [PATCH 20/20] Fix ruff formatting using ruff 0.8.6 --- src/dscim/menu/main_recipe.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/dscim/menu/main_recipe.py b/src/dscim/menu/main_recipe.py index eeadfdab..856c5e02 100644 --- a/src/dscim/menu/main_recipe.py +++ b/src/dscim/menu/main_recipe.py @@ -247,17 +247,17 @@ def __init__( self.logger = logging.getLogger(__name__) if self.quantreg_quantiles is not None: - assert len(self.quantreg_quantiles) == len(self.quantreg_weights), ( - "Length of quantreg quantiles does not match length of weights." - ) + assert len(self.quantreg_quantiles) == len( + self.quantreg_weights + ), "Length of quantreg quantiles does not match length of weights." - assert self.discounting_type in self.DISCOUNT_TYPES, ( - f"Discount type not implemented. Try one of {self.DISCOUNT_TYPES}." - ) + assert ( + self.discounting_type in self.DISCOUNT_TYPES + ), f"Discount type not implemented. Try one of {self.DISCOUNT_TYPES}." - assert self.formula in self.FORMULAS, ( - f"Formula not implemented. Try one of {self.FORMULAS}." - ) + assert ( + self.formula in self.FORMULAS + ), f"Formula not implemented. Try one of {self.FORMULAS}." # Set stream of discounts to None if discounting_type is 'constant' # 'constant_model_collapsed' should be here except that we allow