diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a39571a1b..f5763de4bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y pkg-config build-essential - sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev + sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev - name: Install Python dependencies run: | @@ -200,7 +200,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y pkg-config build-essential - sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev + sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev sudo apt-get install -y bash zsh fish # for shell completion tests sudo apt-get install -y rclone openssh-server curl if [[ "$TOXENV" == *"llfuse"* ]]; then @@ -435,7 +435,7 @@ jobs: freebsd) export IGNORE_OSVERSION=yes sudo -E pkg update -f - sudo -E pkg install -y xxhash liblz4 zstd pkgconf + sudo -E pkg install -y xxhash liblz4 pkgconf sudo -E pkg install -y fusefs-libs sudo -E kldload fusefs sudo -E sysctl vfs.usermount=1 @@ -491,7 +491,7 @@ jobs: echo "https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null sudo -E pkgin update sudo -E pkgin -y upgrade - sudo -E pkgin -y install zstd lz4 xxhash git + sudo -E pkgin -y install lz4 xxhash git sudo -E pkgin -y install rust sudo -E pkgin -y install pkg-config sudo -E pkgin -y install py311-pip py311-virtualenv py311-tox @@ -525,7 +525,7 @@ jobs: ;; openbsd) - sudo -E pkg_add xxhash lz4 zstd git + sudo -E pkg_add xxhash lz4 git sudo -E pkg_add rust sudo -E pkg_add openssl%3.4 sudo -E pkg_add py3-pip py3-virtualenv py3-tox @@ -563,12 +563,12 @@ jobs: haiku) pkgman refresh - pkgman install -y git pkgconfig zstd lz4 xxhash + pkgman install -y git pkgconfig lz4 xxhash pkgman install -y openssl3 pkgman install -y rust_bin pkgman install -y python3.10 pkgman install -y cffi - pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel + pkgman install -y lz4_devel xxhash_devel openssl3_devel libffi_devel # there is no pkgman package for tox, so we install it into a venv python3 -m ensurepip --upgrade @@ -578,7 +578,6 @@ jobs: export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}" export BORG_LIBLZ4_PREFIX=/system/develop - export BORG_LIBZSTD_PREFIX=/system/develop export BORG_LIBXXHASH_PREFIX=/system/develop export BORG_OPENSSL_PREFIX=/system/develop pip install -r requirements.d/development.txt diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index cda0408933..f9404e196e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -66,7 +66,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y pkg-config build-essential - sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev + sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v4 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 784ad2981a..390028f896 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -16,7 +16,6 @@ build: - libacl1-dev - libssl-dev - liblz4-dev - - libzstd-dev - libxxhash-dev python: diff --git a/Brewfile b/Brewfile index 063f59f8c1..f1fbecc138 100644 --- a/Brewfile +++ b/Brewfile @@ -1,5 +1,4 @@ brew 'pkgconf' -brew 'zstd' brew 'lz4' brew 'xxhash' brew 'openssl@3' diff --git a/Vagrantfile b/Vagrantfile index 578700821e..312467e332 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -16,7 +16,7 @@ def packages_debianoid(user) apt-get -y -qq dist-upgrade # for building borgbackup and dependencies: apt install -y pkg-config - apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev || true + apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev || true apt install -y libfuse-dev fuse || true apt install -y libfuse3-dev fuse3 || true apt install -y locales || true @@ -38,7 +38,7 @@ def packages_freebsd # install all the (security and other) updates, base system freebsd-update --not-running-from-cron fetch install # for building borgbackup and dependencies: - pkg install -y xxhash liblz4 zstd pkgconf + pkg install -y xxhash liblz4 pkgconf pkg install -y fusefs-libs || true pkg install -y fusefs-libs3 || true pkg install -y rust @@ -85,7 +85,6 @@ def packages_openbsd chsh -s bash vagrant pkg_add xxhash pkg_add lz4 - pkg_add zstd pkg_add git # no fakeroot pkg_add rust pkg_add openssl%3.4 @@ -100,7 +99,7 @@ def packages_netbsd echo 'https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/$arch/9.3/All' > /usr/pkg/etc/pkgin/repositories.conf pkgin update pkgin -y upgrade - pkg_add zstd lz4 xxhash git + pkg_add lz4 xxhash git pkg_add rust pkg_add bash chsh -s bash vagrant diff --git a/docs/installation.rst b/docs/installation.rst index 535ebef87c..143533f7b2 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -166,7 +166,6 @@ development header files (sometimes in a separate `-dev` or `-devel` package). * libacl_ (which depends on libattr_) * libxxhash_ >= 0.8.1 * liblz4_ >= 1.7.0 (r129) -* libzstd_ >= 1.3.0 * libffi (required for argon2-cffi-bindings) * pkg-config (cli tool) - Borg uses this to discover header and library locations automatically. Alternatively, you can also point to them via some @@ -201,7 +200,7 @@ Arch Linux Install the runtime and build dependencies:: - pacman -S python python-pip python-virtualenv openssl acl xxhash lz4 zstd base-devel + pacman -S python python-pip python-virtualenv openssl acl xxhash lz4 base-devel pacman -S fuse2 # needed for llfuse pacman -S fuse3 # needed for pyfuse3 @@ -217,7 +216,7 @@ Install the dependencies with development headers:: sudo apt-get install python3 python3-dev python3-pip python3-virtualenv \ libacl1-dev \ libssl-dev \ - liblz4-dev libzstd-dev libxxhash-dev \ + liblz4-dev libxxhash-dev \ libffi-dev \ build-essential pkg-config sudo apt-get install libfuse-dev fuse # needed for llfuse @@ -235,7 +234,7 @@ Install the dependencies with development headers:: sudo dnf install python3 python3-devel python3-pip python3-virtualenv \ libacl-devel \ openssl-devel \ - lz4-devel libzstd-devel xxhash-devel \ + lz4-devel xxhash-devel \ libffi-devel \ pkgconf sudo dnf install gcc gcc-c++ redhat-rpm-config @@ -252,7 +251,7 @@ Install the dependencies automatically using zypper:: Alternatively, you can enumerate all build dependencies in the command line:: sudo zypper install python3 python3-devel \ - libacl-devel openssl-devel xxhash-devel libzstd-devel liblz4-devel \ + libacl-devel openssl-devel xxhash-devel liblz4-devel \ libffi-devel \ python3-Cython python3-Sphinx python3-msgpack-python python3-pkgconfig pkgconf \ python3-pytest python3-setuptools python3-setuptools_scm \ @@ -303,7 +302,7 @@ and commands to make FUSE work for using the mount command. pkg install -y python3 pkgconf pkg install openssl - pkg install liblz4 zstd xxhash + pkg install liblz4 xxhash pkg install fusefs-libs # needed for llfuse pkg install -y git python3 -m ensurepip # to install pip for Python3 @@ -347,7 +346,7 @@ Use the Cygwin installer to install the dependencies:: python39 python39-devel python39-setuptools python39-pip python39-wheel python39-virtualenv - libssl-devel libxxhash-devel liblz4-devel libzstd-devel + libssl-devel libxxhash-devel liblz4-devel binutils gcc-g++ git make openssh Make sure to use a virtual environment to avoid confusions with any Python installed on Windows. diff --git a/docs/usage/general/environment.rst.inc b/docs/usage/general/environment.rst.inc index ce80aabf70..1765b44738 100644 --- a/docs/usage/general/environment.rst.inc +++ b/docs/usage/general/environment.rst.inc @@ -218,9 +218,6 @@ Building: BORG_LIBLZ4_PREFIX Adds given prefix directory to the default locations. If a 'include/lz4.h' is found Borg will be linked against the system liblz4 instead of a bundled implementation. (setup.py) - BORG_LIBZSTD_PREFIX - Adds given prefix directory to the default locations. If a 'include/zstd.h' is found Borg - will be linked against the system libzstd instead of a bundled implementation. (setup.py) Please note: diff --git a/pyproject.toml b/pyproject.toml index e828a0e8cf..d5e0187345 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'", # for others: 2.6+ works consistently. "argon2-cffi", "shtab>=1.8.0", + "backports-zstd; python_version < '3.14'", # for python < 3.14. ] [project.optional-dependencies] diff --git a/scripts/Dockerfile.linux-run b/scripts/Dockerfile.linux-run index 7d43792456..f6a1220cf4 100644 --- a/scripts/Dockerfile.linux-run +++ b/scripts/Dockerfile.linux-run @@ -10,7 +10,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libacl1-dev \ libxxhash-dev \ liblz4-dev \ - libzstd-dev \ libfuse3-dev \ fuse3 \ python3-dev \ diff --git a/scripts/msys2-install-deps b/scripts/msys2-install-deps index 963754fa6a..6e6836dd28 100644 --- a/scripts/msys2-install-deps +++ b/scripts/msys2-install-deps @@ -1,6 +1,6 @@ #!/bin/bash -pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,zstd,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko} +pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko} if [ "$1" = "development" ]; then pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist} diff --git a/setup.py b/setup.py index acb80596f4..033d2dfa43 100644 --- a/setup.py +++ b/setup.py @@ -160,7 +160,6 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s compress_ext_kwargs = members_appended( dict(sources=[compress_source]), lib_ext_kwargs(pc, "BORG_LIBLZ4_PREFIX", "lz4", "liblz4", ">= 1.7.0"), - lib_ext_kwargs(pc, "BORG_LIBZSTD_PREFIX", "zstd", "libzstd", ">= 1.3.0"), dict(extra_compile_args=cflags), ) diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index 3327f4ee12..ba7d124b32 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -29,6 +29,12 @@ except ImportError: from .constants import MAX_DATA_SIZE, ROBJ_FILE_STREAM from .helpers import Buffer, DecompressionError +import sys + +if sys.version_info >= (3, 14): + from compression import zstd +else: + from backports import zstd cdef extern from "lz4.h": @@ -36,21 +42,8 @@ cdef extern from "lz4.h": int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil int LZ4_compressBound(int inputSize) nogil - -cdef extern from "zstd.h": - size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) nogil - size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t compressedSize) nogil - size_t ZSTD_compressBound(size_t srcSize) nogil - unsigned long long ZSTD_CONTENTSIZE_UNKNOWN - unsigned long long ZSTD_CONTENTSIZE_ERROR - unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) nogil - unsigned ZSTD_isError(size_t code) nogil - const char* ZSTD_getErrorName(size_t code) nogil - - buffer = Buffer(bytearray, size=0) - cdef class CompressorBase: """ base class for all (de)compression classes, @@ -303,12 +296,10 @@ class LZMA(DecidingCompressor): except lzma.LZMAError as e: raise DecompressionError(str(e)) from None - class ZSTD(DecidingCompressor): - """zstd compression / decompression (pypi: zstandard, gh: python-zstandard)""" - # This is a NOT THREAD SAFE implementation. - # Only ONE python context must be created at a time. - # It should work flawlessly as long as borg will call ONLY ONE compression job at time. + """ + zstd compression / decompression (python stdlib (python >= 3.14)) + """ ID = 0x03 name = 'zstd' @@ -316,61 +307,21 @@ class ZSTD(DecidingCompressor): super().__init__(level=level, legacy_mode=legacy_mode, **kwargs) self.level = level - def _decide(self, meta, idata): - """ - Decides what to do with *data*. Returns (compressor, zstd_data). - - *zstd_data* is the ZSTD result if *compressor* is ZSTD as well, otherwise it is None. - """ - if not isinstance(idata, bytes): - idata = bytes(idata) # code below does not work with memoryview - cdef int isize = len(idata) - cdef int osize - cdef char *source = idata - cdef char *dest - cdef int level = self.level - osize = ZSTD_compressBound(isize) - buf = buffer.get(osize) - dest = buf - with nogil: - osize = ZSTD_compress(dest, osize, source, isize, level) - if ZSTD_isError(osize): - raise Exception('zstd compress failed: %s' % ZSTD_getErrorName(osize)) - # only compress if the result actually is smaller - if osize < isize: - return self, (meta, dest[:osize]) + def _decide(self, meta, data): + zstd_data = zstd.compress(data, self.level) + if len(zstd_data) < len(data): + return self, (meta, zstd_data) else: return NONE_COMPRESSOR, (meta, None) - + def decompress(self, meta, data): - meta, idata = super().decompress(meta, data) - if not isinstance(idata, bytes): - idata = bytes(idata) # code below does not work with memoryview - cdef int isize = len(idata) - cdef unsigned long long osize - cdef unsigned long long rsize - cdef char *source = idata - cdef char *dest - osize = ZSTD_getFrameContentSize(source, isize) - if osize == ZSTD_CONTENTSIZE_ERROR: - raise DecompressionError('zstd get size failed: data was not compressed by zstd') - if osize == ZSTD_CONTENTSIZE_UNKNOWN: - raise DecompressionError('zstd get size failed: original size unknown') + meta, data = super().decompress(meta, data) try: - buf = buffer.get(osize) - except MemoryError: - raise DecompressionError('MemoryError') - dest = buf - with nogil: - rsize = ZSTD_decompress(dest, osize, source, isize) - if ZSTD_isError(rsize): - raise DecompressionError('zstd decompress failed: %s' % ZSTD_getErrorName(rsize)) - if rsize != osize: - raise DecompressionError('zstd decompress failed: size mismatch') - data = dest[:osize] - self.check_fix_size(meta, data) - return meta, data - + data = zstd.decompress(data) + self.check_fix_size(meta, data) + return meta, data + except zstd.ZstdError as e: + raise DecompressionError(str(e)) from None class ZLIB(DecidingCompressor): """