From 39f6be560953b7e17af87e10883e557af25d9519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Flemstr=C3=B6m?= Date: Wed, 31 Aug 2022 08:43:37 +0200 Subject: [PATCH 1/4] #193- Make Equi7 gridding optional --- src/openeo_processes/cubes.py | 4 ++-- src/openeo_processes/utils.py | 35 +++++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/openeo_processes/cubes.py b/src/openeo_processes/cubes.py index f5fbf706..c7946230 100644 --- a/src/openeo_processes/cubes.py +++ b/src/openeo_processes/cubes.py @@ -248,7 +248,7 @@ def exec_xar(data, output_filepath='out', format='GTiff', options={}, write_prod first_data_var = data.data_vars[list(data.data_vars.keys())[0]] data.attrs["crs"] = first_data_var.geobox.crs.to_wkt() - tiles, gridder = get_equi7_tiles(data) + # Renaming the time dimension if 'time' in data.dims: @@ -275,7 +275,7 @@ def exec_xar(data, output_filepath='out', format='GTiff', options={}, write_prod else: ext = 'tif' - final_datasets, dataset_filenames = derive_datasets_and_filenames_from_tiles(gridder, times, datasets, tiles, output_filepath, ext) + final_datasets, dataset_filenames = derive_datasets_and_filenames_from_tiles(times,data, datasets, output_filepath, ext) if (len(final_datasets) == 0) or (len(dataset_filenames) == 0): raise Exception("No tiles could be derived from given dataset") diff --git a/src/openeo_processes/utils.py b/src/openeo_processes/utils.py index 54f01357..f1786150 100644 --- a/src/openeo_processes/utils.py +++ b/src/openeo_processes/utils.py @@ -1,15 +1,15 @@ -import os import functools +import os import re -from datetime import timezone, timedelta, datetime -from typing import Any, Callable, Tuple, List +from datetime import datetime, timedelta, timezone +from typing import Any, Callable, List, Tuple import dask import dask.dataframe as dd -import numpy as np -import xarray as xr import geopandas as gpd +import numpy as np import rasterio +import xarray as xr # This is a workaround for this package now requiring gdal, which isn't straightforward to install with pip. # TODO: Remove this once we've figured out how to properly integrate the gdal dependency for this library @@ -22,6 +22,7 @@ except ImportError: osr = None + def eval_datatype(data): """ @@ -396,8 +397,22 @@ def get_equi7_tiles(data: xr.Dataset): return tiles, gridder -def derive_datasets_and_filenames_from_tiles(gridder, times: List[str], datasets: List[xr.Dataset], - tiles: List[str], output_filepath: str, ext: str): +def derive_datasets_and_filenames_without_tiles(times: List[str], datasets: List[xr.Dataset], + output_filepath: str, ext: str): + final_datasets = [] + dataset_filenames = [] + + for idx, time in enumerate(times): + dataset = datasets[idx] + file_time = np.datetime_as_string(time)[:19].replace('-', '_').replace(':', '_') + temp_file = output_filepath + '_{}.{}'.format(file_time, ext) + final_datasets.append(dataset) + dataset_filenames.append(temp_file) + + return final_datasets, dataset_filenames + +def derive_datasets_and_filenames_from_tiles(times: List[str], data, datasets: List[xr.Dataset], + output_filepath: str, ext: str): """ A function taking an xarray.Dataset and returning a list of EQUI7 Tiles at the relevant resolution layer along with an EQUI7Grid object. @@ -416,9 +431,13 @@ def derive_datasets_and_filenames_from_tiles(gridder, times: List[str], datasets list[xarray.Dataset]: The resulting datasets split across the EQUI7 tile grid. list[str]: The list of filepaths split across the EQUI7 tile grid, corresponding to the list of datasets. """ + if equi7grid is None: + return derive_datasets_and_filenames_without_tiles(times, datasets, output_filepath, ext) + final_datasets = [] dataset_filenames = [] - + + tiles, gridder = get_equi7_tiles(data) for idx, time in enumerate(times): dataset = datasets[idx] file_time = np.datetime_as_string(time)[:19].replace('-', '_').replace(':', '_') From f05ac715dcd7529ef9caa96aac6ff4f217652a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Flemstro=CC=88m?= Date: Mon, 5 Sep 2022 07:53:06 +0200 Subject: [PATCH 2/4] removed Equi7grid for DES --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e7f6f8b4..22523e68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ pyproj = "^3" xgboost = "^1.5.0" rioxarray = "^0.9" geopandas = "^0.10" -Equi7Grid = "^0.1.0" +#Equi7Grid = "^0.1.0" datacube = "^1.8.4" dask-geopandas = "^v0.1.0a7" dask = {extras = ["array"], version = "^2022.02.1"} From 3bc18f2856785ac50f4f4ebaefde9064ffbccfe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Flemstro=CC=88m?= Date: Mon, 5 Sep 2022 08:03:19 +0200 Subject: [PATCH 3/4] removed Equi7grid for DES from lock file as well --- poetry.lock | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/poetry.lock b/poetry.lock index ee851080..63b281de 100644 --- a/poetry.lock +++ b/poetry.lock @@ -233,21 +233,6 @@ tornado = ">=6.0.3" urllib3 = "*" zict = ">=0.1.3" -[[package]] -name = "equi7grid" -version = "0.1.0" -description = "Definition and tools for Equi7Grid and its tiling system." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -geographiclib = "*" -pytileproj = ">=0.0.16" - -[package.extras] -test = ["pytest-cov", "pytest", "nose"] - [[package]] name = "fiona" version = "1.8.21" From cd57592db689f20b238ed7998c947041da809eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Flemstro=CC=88m?= Date: Mon, 5 Sep 2022 08:42:53 +0200 Subject: [PATCH 4/4] removed Equi7grid for DES from lock file as well, stray entry removed --- poetry.lock | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 63b281de..73969c69 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1199,9 +1199,7 @@ distributed = [ {file = "distributed-2022.6.0-py3-none-any.whl", hash = "sha256:80c63a1ee3efb38c6f8634daaaa00f7992b246914a003202b687cedd6299b9c9"}, {file = "distributed-2022.6.0.tar.gz", hash = "sha256:2050d7cf9479766a6f4ad9cf85e97dee548401a19f8bb97af6a73dda9ffd3d05"}, ] -equi7grid = [ - {file = "Equi7Grid-0.1.0.tar.gz", hash = "sha256:82cdedba3ad098be4dfb5ff5971262a1272c95b31b4a9f34e20469287cfe5203"}, -] + fiona = [ {file = "Fiona-1.8.21-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:39c656421e25b4d0d73d0b6acdcbf9848e71f3d9b74f44c27d2d516d463409ae"}, {file = "Fiona-1.8.21-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43b1d2e45506e56cf3a9f59ba5d6f7981f3f75f4725d1e6cb9a33ba856371ebd"},