From 9c2211ab8b12edb9fe9a78274a3d708de866e7d5 Mon Sep 17 00:00:00 2001 From: Jeeva Date: Mon, 9 Dec 2024 16:31:23 +0530 Subject: [PATCH 1/6] get stress table output from OVERVIEW.OUT. Now only run 1 is handled. --- DSSATTools/run.py | 106 +++++++++++++++++++++++++++++++--------------- 1 file changed, 72 insertions(+), 34 deletions(-) diff --git a/DSSATTools/run.py b/DSSATTools/run.py index 028037f..6b4accb 100644 --- a/DSSATTools/run.py +++ b/DSSATTools/run.py @@ -34,10 +34,10 @@ from DSSATTools.base.sections import TabularSubsection OS = platform.system().lower() -OUTPUTS = ['PlantGro', "Weather", "SoilWat", "SoilOrg", "SoilNi"] +OUTPUTS = ['PlantGro', "Weather", "SoilWat", "SoilOrg", "SoilNi", "OVERVIEW", "Summary"] OUTPUT_MAP = { "PlantGro": "GROUT", "SoilWat": "WAOUT", "SoilOrg": "CAOUT", - "Weather": "GROUT", "SoilNi": "NIOUT" + "Weather": "GROUT", "SoilNi": "NIOUT", "Overview":"OVVEW", "Summary":"SUMRY" } SOIL_LAYER_OUTPUTS = ["SoilNi"] @@ -271,41 +271,79 @@ def run(self, for file in self.OUTPUT_LIST: assert f'{file}.OUT' in OUTPUT_FILES, \ f'{file}.OUT does not exist in {self._RUN_PATH}' - table_start = -1 - init_lines = [] - with open(os.path.join(self._RUN_PATH, f'{file}.OUT'), 'r', encoding='cp437') as f: - while True: - table_start += 1 - init_lines.append(f.readline()) - if '@' in init_lines[-1][:10]: - break - - try: - df = pd.read_csv( - os.path.join(self._RUN_PATH, f'{file}.OUT'), - skiprows=table_start, sep=' ', skipinitialspace=True - ) - df = df.dropna(how="all", axis=1) - - except UnicodeDecodeError: + if "OVERVIEW.OUT" in OUTPUT_FILES: + df=self._get_overview_stress() + self._output[file] = df + else: + table_start = -1 + init_lines = [] with open(os.path.join(self._RUN_PATH, f'{file}.OUT'), 'r', encoding='cp437') as f: - lines = f.readlines() - with open(os.path.join(self._RUN_PATH, f'{file}.OUT'), 'w', encoding='utf-8') as f: - f.writelines(lines[table_start:]) - df = pd.read_csv( - os.path.join(self._RUN_PATH, f'{file}.OUT'), - skiprows=0, sep=' ', skipinitialspace=True - ) - if all(('@YEAR' in df.columns, 'DOY' in df.columns)): - df['DOY'] = df.DOY.astype(int).map(lambda x: f'{x:03d}') - df['@YEAR'] = df['@YEAR'].astype(str) - df.index = pd.to_datetime( - (df['@YEAR'] + df['DOY']), - format='%Y%j' - ) - self._output[file] = df + while True: + table_start += 1 + init_lines.append(f.readline()) + if '@' in init_lines[-1][:10]: + break + + try: + df = pd.read_csv( + os.path.join(self._RUN_PATH, f'{file}.OUT'), + skiprows=table_start, sep=' ', skipinitialspace=True + ) + df = df.dropna(how="all", axis=1) + + except UnicodeDecodeError: + with open(os.path.join(self._RUN_PATH, f'{file}.OUT'), 'r', encoding='cp437') as f: + lines = f.readlines() + with open(os.path.join(self._RUN_PATH, f'{file}.OUT'), 'w', encoding='utf-8') as f: + f.writelines(lines[table_start:]) + df = pd.read_csv( + os.path.join(self._RUN_PATH, f'{file}.OUT'), + skiprows=0, sep=' ', skipinitialspace=True + ) + if all(('@YEAR' in df.columns, 'DOY' in df.columns)): + df['DOY'] = df.DOY.astype(int).map(lambda x: f'{x:03d}') + df['@YEAR'] = df['@YEAR'].astype(str) + df.index = pd.to_datetime( + (df['@YEAR'] + df['DOY']), + format='%Y%j' + ) + self._output[file] = df return + + def _get_overview_stress(self): + table_start = -1 + init_lines=[] + with open(os.path.join(self._RUN_PATH, 'OVERVIEW.OUT'), 'r', encoding='cp437') as f: + while True: + table_start += 1 + search_term="DATE" + data = [] + init_lines.append(f.readline()) + if search_term in init_lines[-1][:10]: + header = ['DATE', 'CROP AGE', 'GROWTH STAGE', 'BIOMASS', 'LEAF LAI', \ + 'LEAF NUM', 'N_KG', 'N_PERCENT', 'STRESS H2O', 'STRESS Nitr', \ + 'STRESS Phos1', 'STRESS Phos2', 'RSTG'] + break + # Skip the underline or separator line + f.readline() + for line in f: + if re.match(r'^\s*$', line): # Check for an empty line + break + else: + data.append(line.strip()) + widths = [7, 5, 11, 8, 7, 6, 5, 5, 6, 6, 6, 6, 6] + df_data=[] + for line in data: + columns = [] + start = 0 + for width in widths: + columns.append(line[start:start + width].strip()) # Extract and strip + start += width + df_data.append(columns) + df = pd.DataFrame(df_data, columns=header).dropna(how="all", axis=1) + return df + def close(self): ''' From 5b345586566f4cf96757458896dae0e7c086b497 Mon Sep 17 00:00:00 2001 From: Jeevanand Palanisamy Date: Thu, 12 Dec 2024 11:58:56 +0530 Subject: [PATCH 2/6] Update run.py corrected a small mistake where i had if condition wrong. --- DSSATTools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DSSATTools/run.py b/DSSATTools/run.py index 6b4accb..b4f519e 100644 --- a/DSSATTools/run.py +++ b/DSSATTools/run.py @@ -271,7 +271,7 @@ def run(self, for file in self.OUTPUT_LIST: assert f'{file}.OUT' in OUTPUT_FILES, \ f'{file}.OUT does not exist in {self._RUN_PATH}' - if "OVERVIEW.OUT" in OUTPUT_FILES: + if "OVERVIEW" in file: df=self._get_overview_stress() self._output[file] = df else: From 2bf12d4001de975292e9557508fa4ad4cd4b0278 Mon Sep 17 00:00:00 2001 From: Diego Quintero Date: Fri, 20 Dec 2024 15:16:51 -0500 Subject: [PATCH 3/6] Fixed ecotype lines in Maize --- DSSATTools/static/Genotype/MZCER048.ECO | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DSSATTools/static/Genotype/MZCER048.ECO b/DSSATTools/static/Genotype/MZCER048.ECO index 290d6b0..96c9c4d 100644 --- a/DSSATTools/static/Genotype/MZCER048.ECO +++ b/DSSATTools/static/Genotype/MZCER048.ECO @@ -25,8 +25,8 @@ IB0002 GENERIC MIDWEST2 8.0 34.0 34.0 12.5 4.0 6.0 170. 4.5 0.85 IB0003 GENERIC MIDWEST3 8.0 34.0 34.0 12.5 4.0 6.0 170. 2.0 0.85 IB0004 +5% RUE MIDWEST1 8.0 34.0 34.0 12.5 4.0 6.0 170. 4.4 0.85 DFAULT DEFAULT 8.0 34.0 34.0 12.5 4.0 6.0 170. 4.2 0.85 -IB0011 GENERIC MIDWEST1 8.0 34.0 34.0 12.5 4.0 6.0 170. 4.0 0.85 #Good KE 24 May 2024 -IB0012 GENERIC MIDWEST1 8.0 34.0 34.0 12.5 4.0 6.0 170. 4.0 0.85 #Good KE 24 May 2024 -IB0013 GENERIC MIDWEST1 8.0 34.0 34.0 12.5 2.0 6.0 100. 7.0 0.85 #Good KE 24 May 2024 +IB0011 GENERIC MIDWEST1 8.0 34.0 34.0 12.5 4.0 6.0 170. 4.0 0.85 +IB0012 GENERIC MIDWEST1 8.0 34.0 34.0 12.5 4.0 6.0 170. 4.0 0.85 +IB0013 GENERIC MIDWEST1 8.0 34.0 34.0 12.5 2.0 6.0 100. 7.0 0.85 From c8e937e20b7f4839d3bc0231508b49b348779a6a Mon Sep 17 00:00:00 2001 From: Jeeva Date: Mon, 9 Dec 2024 16:31:23 +0530 Subject: [PATCH 4/6] get stress table output from OVERVIEW.OUT. Now only run 1 is handled. --- DSSATTools/run.py | 106 +++++++++++++++++++++++++++++++--------------- 1 file changed, 72 insertions(+), 34 deletions(-) diff --git a/DSSATTools/run.py b/DSSATTools/run.py index 028037f..6b4accb 100644 --- a/DSSATTools/run.py +++ b/DSSATTools/run.py @@ -34,10 +34,10 @@ from DSSATTools.base.sections import TabularSubsection OS = platform.system().lower() -OUTPUTS = ['PlantGro', "Weather", "SoilWat", "SoilOrg", "SoilNi"] +OUTPUTS = ['PlantGro', "Weather", "SoilWat", "SoilOrg", "SoilNi", "OVERVIEW", "Summary"] OUTPUT_MAP = { "PlantGro": "GROUT", "SoilWat": "WAOUT", "SoilOrg": "CAOUT", - "Weather": "GROUT", "SoilNi": "NIOUT" + "Weather": "GROUT", "SoilNi": "NIOUT", "Overview":"OVVEW", "Summary":"SUMRY" } SOIL_LAYER_OUTPUTS = ["SoilNi"] @@ -271,41 +271,79 @@ def run(self, for file in self.OUTPUT_LIST: assert f'{file}.OUT' in OUTPUT_FILES, \ f'{file}.OUT does not exist in {self._RUN_PATH}' - table_start = -1 - init_lines = [] - with open(os.path.join(self._RUN_PATH, f'{file}.OUT'), 'r', encoding='cp437') as f: - while True: - table_start += 1 - init_lines.append(f.readline()) - if '@' in init_lines[-1][:10]: - break - - try: - df = pd.read_csv( - os.path.join(self._RUN_PATH, f'{file}.OUT'), - skiprows=table_start, sep=' ', skipinitialspace=True - ) - df = df.dropna(how="all", axis=1) - - except UnicodeDecodeError: + if "OVERVIEW.OUT" in OUTPUT_FILES: + df=self._get_overview_stress() + self._output[file] = df + else: + table_start = -1 + init_lines = [] with open(os.path.join(self._RUN_PATH, f'{file}.OUT'), 'r', encoding='cp437') as f: - lines = f.readlines() - with open(os.path.join(self._RUN_PATH, f'{file}.OUT'), 'w', encoding='utf-8') as f: - f.writelines(lines[table_start:]) - df = pd.read_csv( - os.path.join(self._RUN_PATH, f'{file}.OUT'), - skiprows=0, sep=' ', skipinitialspace=True - ) - if all(('@YEAR' in df.columns, 'DOY' in df.columns)): - df['DOY'] = df.DOY.astype(int).map(lambda x: f'{x:03d}') - df['@YEAR'] = df['@YEAR'].astype(str) - df.index = pd.to_datetime( - (df['@YEAR'] + df['DOY']), - format='%Y%j' - ) - self._output[file] = df + while True: + table_start += 1 + init_lines.append(f.readline()) + if '@' in init_lines[-1][:10]: + break + + try: + df = pd.read_csv( + os.path.join(self._RUN_PATH, f'{file}.OUT'), + skiprows=table_start, sep=' ', skipinitialspace=True + ) + df = df.dropna(how="all", axis=1) + + except UnicodeDecodeError: + with open(os.path.join(self._RUN_PATH, f'{file}.OUT'), 'r', encoding='cp437') as f: + lines = f.readlines() + with open(os.path.join(self._RUN_PATH, f'{file}.OUT'), 'w', encoding='utf-8') as f: + f.writelines(lines[table_start:]) + df = pd.read_csv( + os.path.join(self._RUN_PATH, f'{file}.OUT'), + skiprows=0, sep=' ', skipinitialspace=True + ) + if all(('@YEAR' in df.columns, 'DOY' in df.columns)): + df['DOY'] = df.DOY.astype(int).map(lambda x: f'{x:03d}') + df['@YEAR'] = df['@YEAR'].astype(str) + df.index = pd.to_datetime( + (df['@YEAR'] + df['DOY']), + format='%Y%j' + ) + self._output[file] = df return + + def _get_overview_stress(self): + table_start = -1 + init_lines=[] + with open(os.path.join(self._RUN_PATH, 'OVERVIEW.OUT'), 'r', encoding='cp437') as f: + while True: + table_start += 1 + search_term="DATE" + data = [] + init_lines.append(f.readline()) + if search_term in init_lines[-1][:10]: + header = ['DATE', 'CROP AGE', 'GROWTH STAGE', 'BIOMASS', 'LEAF LAI', \ + 'LEAF NUM', 'N_KG', 'N_PERCENT', 'STRESS H2O', 'STRESS Nitr', \ + 'STRESS Phos1', 'STRESS Phos2', 'RSTG'] + break + # Skip the underline or separator line + f.readline() + for line in f: + if re.match(r'^\s*$', line): # Check for an empty line + break + else: + data.append(line.strip()) + widths = [7, 5, 11, 8, 7, 6, 5, 5, 6, 6, 6, 6, 6] + df_data=[] + for line in data: + columns = [] + start = 0 + for width in widths: + columns.append(line[start:start + width].strip()) # Extract and strip + start += width + df_data.append(columns) + df = pd.DataFrame(df_data, columns=header).dropna(how="all", axis=1) + return df + def close(self): ''' From 94f29e9b5dc0718f7f3630924b23138dcda84934 Mon Sep 17 00:00:00 2001 From: Jeevanand Palanisamy Date: Thu, 12 Dec 2024 11:58:56 +0530 Subject: [PATCH 5/6] Update run.py corrected a small mistake where i had if condition wrong. --- DSSATTools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DSSATTools/run.py b/DSSATTools/run.py index 6b4accb..b4f519e 100644 --- a/DSSATTools/run.py +++ b/DSSATTools/run.py @@ -271,7 +271,7 @@ def run(self, for file in self.OUTPUT_LIST: assert f'{file}.OUT' in OUTPUT_FILES, \ f'{file}.OUT does not exist in {self._RUN_PATH}' - if "OVERVIEW.OUT" in OUTPUT_FILES: + if "OVERVIEW" in file: df=self._get_overview_stress() self._output[file] = df else: From 21a3b1b4e3f9f76811ccc7810abacf2eeb9dd0dd Mon Sep 17 00:00:00 2001 From: Diego Quintero Date: Fri, 20 Dec 2024 16:04:30 -0500 Subject: [PATCH 6/6] couple changes to tests and output read functions --- DSSATTools/run.py | 21 ++++++++++++++------- tests/test_run.py | 3 +++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/DSSATTools/run.py b/DSSATTools/run.py index b4f519e..4188cd4 100644 --- a/DSSATTools/run.py +++ b/DSSATTools/run.py @@ -34,10 +34,12 @@ from DSSATTools.base.sections import TabularSubsection OS = platform.system().lower() -OUTPUTS = ['PlantGro', "Weather", "SoilWat", "SoilOrg", "SoilNi", "OVERVIEW", "Summary"] +OUTPUTS = ['PlantGro', "Weather", "SoilWat", "SoilOrg", "SoilNi", + "OVERVIEW", "Summary"] OUTPUT_MAP = { "PlantGro": "GROUT", "SoilWat": "WAOUT", "SoilOrg": "CAOUT", - "Weather": "GROUT", "SoilNi": "NIOUT", "Overview":"OVVEW", "Summary":"SUMRY" + "Weather": "GROUT", "SoilNi": "NIOUT", "OVERVIEW": "OVVEW", + "Summary": "SUMRY" } SOIL_LAYER_OUTPUTS = ["SoilNi"] @@ -229,8 +231,6 @@ def run(self, wth_path = os.path.join(self._RUN_PATH, 'Weather') weather.write(wth_path, management=management) - - with open(os.path.join(self._RUN_PATH, CONFILE), 'w') as f: f.write(f'WED {wth_path}\n') if crop._CODE in ["WH", "BA"]: @@ -264,6 +264,11 @@ def run(self, var for var in OUTPUTS if management.simulation_controls.get(OUTPUT_MAP.get(var)) in ("Y", None) ] + + # There's not overview file for perenial forages + if crop.crop_name in PERENIAL_FORAGES: + self.OUTPUT_LIST.remove("OVERVIEW") + # Check for man.simulation_controls["WATER"] if management.simulation_controls["WATER"] == "N": self.OUTPUT_LIST = list(filter(lambda x: x != "SoilWat", self.OUTPUT_LIST)) @@ -272,8 +277,10 @@ def run(self, assert f'{file}.OUT' in OUTPUT_FILES, \ f'{file}.OUT does not exist in {self._RUN_PATH}' if "OVERVIEW" in file: - df=self._get_overview_stress() - self._output[file] = df + self.stress_table = self._get_overview_stress() + with open(os.path.join(self._RUN_PATH, f'{file}.OUT'), + 'r', encoding='cp437') as f: + self._output[file] = f.readlines() else: table_start = -1 init_lines = [] @@ -320,7 +327,7 @@ def _get_overview_stress(self): search_term="DATE" data = [] init_lines.append(f.readline()) - if search_term in init_lines[-1][:10]: + if search_term in init_lines[-1][:12]: header = ['DATE', 'CROP AGE', 'GROWTH STAGE', 'BIOMASS', 'LEAF LAI', \ 'LEAF NUM', 'N_KG', 'N_PERCENT', 'STRESS H2O', 'STRESS Nitr', \ 'STRESS Phos1', 'STRESS Phos2', 'RSTG'] diff --git a/tests/test_run.py b/tests/test_run.py index c387d40..f62741f 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -786,12 +786,15 @@ def test_outputs(): man = Management( planting_date=DATES[10], ) + man.simulation_controls["OVVEW"] = "N" + man.simulation_controls["SUMRY"] = "N" dssat = DSSAT() dssat.setup(cwd=os.path.join(TMP, 'test_mz')) dssat.run( soil=soil, weather=wth, crop=crop, management=man, ) + outputs = ['PlantGro', "Weather", "SoilWat", "SoilOrg"] assert all(map(lambda x: x in outputs, dssat.output))