Support .tar.gz files from Campina

Author: siiptuo

src/processing/instrument_process.py

@@ -5,6 +5,7 @@
 import logging
 import re
 import shutil
+import tarfile
 from collections import defaultdict
 from pathlib import Path
 from typing import Literal, NoReturn
@@ -160,7 +161,7 @@ def process_mira_10(self) -> None:
 
     def process_mira_35(self) -> None:
         full_paths, self.uuid.raw = self.download_instrument(
-            filename_suffix={".znc", ".znc.gz"},
+            filename_suffix={".znc", ".znc.gz", ".znc.tar.gz"},
             allow_empty=True,
             exclude_pattern=r"[a-z]+\.znc",
         )
@@ -925,11 +926,11 @@ def _get_valid_uuids(
 
 
 def _unzip_gz_files(full_paths: list[Path]) -> list[Path]:
-    paths_out = []
+    paths_out: list[Path] = []
     for path_in in full_paths:
         try:
-            paths_out.append(unzip_gz_file(path_in))
-        except (EOFError, gzip.BadGzipFile) as err:
+            paths_out.extend(unzip_gz_file(path_in))
+        except (EOFError, gzip.BadGzipFile, tarfile.TarError) as err:
             logging.warning("Cannot unzip gz file %s: %s", path_in, err)
     return paths_out
 

src/processing/utils.py

@@ -2,6 +2,8 @@
 import gzip
 import logging
 import shutil
+import tarfile
+from collections.abc import Iterable
 from pathlib import Path
 from typing import Literal
 from uuid import UUID
@@ -220,16 +222,25 @@ def build_file_landing_page_url(uuid: str | UUID) -> str:
     return f"{base}/file/{uuid}"
 
 
-def unzip_gz_file(path_in: Path) -> Path:
+def unzip_gz_file(path_in: Path) -> Iterable[Path]:
     if path_in.suffix != ".gz":
-        return path_in
+        yield path_in
+        return
     path_out = path_in.parent / path_in.stem
     logging.debug(f"Decompressing {path_in} to {path_out}")
     with gzip.open(path_in, "rb") as file_in:
-        with open(path_out, "wb") as file_out:
-            shutil.copyfileobj(file_in, file_out)
+        if path_out.suffix == ".tar":
+            with tarfile.TarFile(fileobj=file_in) as tar:
+                for tarinfo in tar:
+                    info_out = path_in.parent / tarinfo.name
+                    logging.debug("Extracting to %s", info_out)
+                    tar.extract(tarinfo, path_in.parent, filter="data")
+                    yield info_out
+        else:
+            with open(path_out, "wb") as file_out:
+                shutil.copyfileobj(file_in, file_out)
+            yield path_out
     path_in.unlink()
-    return path_out
 
 
 def create_product_put_payload(

tests/unit/test_utils_module.py

@@ -1,3 +1,7 @@
+import gzip
+import shutil
+import tarfile
+from io import BytesIO
 from pathlib import Path
 
 import netCDF4
@@ -9,6 +13,7 @@
 
 from processing import netcdf_comparer
 from processing.netcdf_comparer import NCDiff
+from processing.utils import unzip_gz_file
 
 test_file_path = Path(__file__).parent.absolute()
 
@@ -614,3 +619,42 @@ def test_compression(tmp_path: Path) -> None:
         nc1.createVariable("time", "f8", ("time",), zlib=False)
         nc2.createVariable("time", "f8", ("time",), zlib=True)
     assert netcdf_comparer.nc_difference(old_file, new_file) == NCDiff.MINOR
+
+
+def test_unzip_regular_gz_file(tmp_path: Path) -> None:
+    regular_file = tmp_path / "test.txt"
+    regular_file.write_text("This is a test file.")
+    gz_file = tmp_path / "test.txt.gz"
+    with open(regular_file, "rb") as f_in:
+        with gzip.open(gz_file, "wb") as f_out:
+            shutil.copyfileobj(f_in, f_out)
+
+    result = list(unzip_gz_file(gz_file))
+    assert len(result) == 1
+    assert result[0].name == "test.txt"
+    assert result[0].read_text() == "This is a test file."
+    assert not gz_file.exists()  # Ensure the .gz file is deleted
+
+
+def test_unzip_tar_gz_file(tmp_path: Path) -> None:
+    tar_gz_file = tmp_path / "test.tar.gz"
+    with tarfile.open(tar_gz_file, "w:gz") as tar:
+        info = tarfile.TarInfo("subdir/test.txt")
+        info.size = len(b"This is a test file inside a tar.")
+        tar.addfile(info, fileobj=BytesIO(b"This is a test file inside a tar."))
+
+    result = list(unzip_gz_file(tar_gz_file))
+    assert len(result) == 1
+    assert result[0].name == "test.txt"
+    assert result[0].read_text() == "This is a test file inside a tar."
+    assert not tar_gz_file.exists()  # Ensure the .tar.gz file is deleted
+
+
+def test_unzip_non_gz_file(tmp_path: Path) -> None:
+    non_gz_file = tmp_path / "test.txt"
+    non_gz_file.write_text("This is a non-gz file.")
+
+    result = list(unzip_gz_file(non_gz_file))
+    assert len(result) == 1
+    assert result[0].name == "test.txt"
+    assert result[0].read_text() == "This is a non-gz file."