use zstd from python lib or backports.zstd (python<'3.14'), closes #9261

.github/workflows/ci.yml

@@ -79,7 +79,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y pkg-config build-essential
-        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
+        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev
 
     - name: Install Python dependencies
       run: |
@@ -200,7 +200,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y pkg-config build-essential
-        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
+        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev
         sudo apt-get install -y bash zsh fish  # for shell completion tests
         sudo apt-get install -y rclone openssh-server curl
         if [[ "$TOXENV" == *"llfuse"* ]]; then
@@ -435,7 +435,7 @@ jobs:
               freebsd)
                 export IGNORE_OSVERSION=yes
                 sudo -E pkg update -f
-                sudo -E pkg install -y xxhash liblz4 zstd pkgconf
+                sudo -E pkg install -y xxhash liblz4 pkgconf
                 sudo -E pkg install -y fusefs-libs
                 sudo -E kldload fusefs
                 sudo -E sysctl vfs.usermount=1
@@ -491,7 +491,7 @@ jobs:
                 echo "https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
                 sudo -E pkgin update
                 sudo -E pkgin -y upgrade
-                sudo -E pkgin -y install zstd lz4 xxhash git
+                sudo -E pkgin -y install lz4 xxhash git
                 sudo -E pkgin -y install rust
                 sudo -E pkgin -y install pkg-config
                 sudo -E pkgin -y install py311-pip py311-virtualenv py311-tox
@@ -525,7 +525,7 @@ jobs:
                 ;;
 
               openbsd)
-                sudo -E pkg_add xxhash lz4 zstd git
+                sudo -E pkg_add xxhash lz4 git
                 sudo -E pkg_add rust
                 sudo -E pkg_add openssl%3.4
                 sudo -E pkg_add py3-pip py3-virtualenv py3-tox
@@ -563,12 +563,12 @@ jobs:
 
               haiku)
                 pkgman refresh
-                pkgman install -y git pkgconfig zstd lz4 xxhash
+                pkgman install -y git pkgconfig lz4 xxhash
                 pkgman install -y openssl3
                 pkgman install -y rust_bin
                 pkgman install -y python3.10
                 pkgman install -y cffi
-                pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel
+                pkgman install -y lz4_devel xxhash_devel openssl3_devel libffi_devel
 
                 # there is no pkgman package for tox, so we install it into a venv
                 python3 -m ensurepip --upgrade
@@ -578,7 +578,6 @@ jobs:
 
                 export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
                 export BORG_LIBLZ4_PREFIX=/system/develop
-                export BORG_LIBZSTD_PREFIX=/system/develop
                 export BORG_LIBXXHASH_PREFIX=/system/develop
                 export BORG_OPENSSL_PREFIX=/system/develop
                 pip install -r requirements.d/development.txt

.github/workflows/codeql-analysis.yml

@@ -66,7 +66,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y pkg-config build-essential
-        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
+        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v4

.readthedocs.yaml

@@ -16,7 +16,6 @@ build:
         - libacl1-dev
         - libssl-dev
         - liblz4-dev
-        - libzstd-dev
         - libxxhash-dev
 
 python:

Brewfile

@@ -1,5 +1,4 @@
 brew 'pkgconf'
-brew 'zstd'
 brew 'lz4'
 brew 'xxhash'
 brew 'openssl@3'

Vagrantfile

@@ -16,7 +16,7 @@ def packages_debianoid(user)
     apt-get -y -qq dist-upgrade
     # for building borgbackup and dependencies:
     apt install -y pkg-config
-    apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev || true
+    apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev || true
     apt install -y libfuse-dev fuse || true
     apt install -y libfuse3-dev fuse3 || true
     apt install -y locales || true
@@ -38,7 +38,7 @@ def packages_freebsd
     # install all the (security and other) updates, base system
     freebsd-update --not-running-from-cron fetch install
     # for building borgbackup and dependencies:
-    pkg install -y xxhash liblz4 zstd pkgconf
+    pkg install -y xxhash liblz4 pkgconf
     pkg install -y fusefs-libs || true
     pkg install -y fusefs-libs3 || true
     pkg install -y rust
@@ -85,7 +85,6 @@ def packages_openbsd
     chsh -s bash vagrant
     pkg_add xxhash
     pkg_add lz4
-    pkg_add zstd
     pkg_add git  # no fakeroot
     pkg_add rust
     pkg_add openssl%3.4
@@ -100,7 +99,7 @@ def packages_netbsd
     echo 'https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/$arch/9.3/All' > /usr/pkg/etc/pkgin/repositories.conf
     pkgin update
     pkgin -y upgrade
-    pkg_add zstd lz4 xxhash git
+    pkg_add lz4 xxhash git
     pkg_add rust
     pkg_add bash
     chsh -s bash vagrant

docs/installation.rst

@@ -166,7 +166,6 @@ development header files (sometimes in a separate `-dev` or `-devel` package).
 * libacl_ (which depends on libattr_)
 * libxxhash_ >= 0.8.1
 * liblz4_ >= 1.7.0 (r129)
-* libzstd_ >= 1.3.0
 * libffi (required for argon2-cffi-bindings)
 * pkg-config (cli tool) - Borg uses this to discover header and library
   locations automatically. Alternatively, you can also point to them via some
@@ -201,7 +200,7 @@ Arch Linux
 
 Install the runtime and build dependencies::
 
-    pacman -S python python-pip python-virtualenv openssl acl xxhash lz4 zstd base-devel
+    pacman -S python python-pip python-virtualenv openssl acl xxhash lz4 base-devel
     pacman -S fuse2     # needed for llfuse
     pacman -S fuse3     # needed for pyfuse3
 
@@ -217,7 +216,7 @@ Install the dependencies with development headers::
     sudo apt-get install python3 python3-dev python3-pip python3-virtualenv \
     libacl1-dev \
     libssl-dev \
-    liblz4-dev libzstd-dev libxxhash-dev \
+    liblz4-dev libxxhash-dev \
     libffi-dev \
     build-essential pkg-config
     sudo apt-get install libfuse-dev fuse    # needed for llfuse
@@ -235,7 +234,7 @@ Install the dependencies with development headers::
     sudo dnf install python3 python3-devel python3-pip python3-virtualenv \
     libacl-devel \
     openssl-devel \
-    lz4-devel libzstd-devel xxhash-devel \
+    lz4-devel xxhash-devel \
     libffi-devel \
     pkgconf
     sudo dnf install gcc gcc-c++ redhat-rpm-config
@@ -252,7 +251,7 @@ Install the dependencies automatically using zypper::
 Alternatively, you can enumerate all build dependencies in the command line::
 
     sudo zypper install python3 python3-devel \
-    libacl-devel openssl-devel xxhash-devel libzstd-devel liblz4-devel \
+    libacl-devel openssl-devel xxhash-devel liblz4-devel \
     libffi-devel \
     python3-Cython python3-Sphinx python3-msgpack-python python3-pkgconfig pkgconf \
     python3-pytest python3-setuptools python3-setuptools_scm \
@@ -303,7 +302,7 @@ and commands to make FUSE work for using the mount command.
 
      pkg install -y python3 pkgconf
      pkg install openssl
-     pkg install liblz4 zstd xxhash
+     pkg install liblz4 xxhash
      pkg install fusefs-libs  # needed for llfuse
      pkg install -y git
      python3 -m ensurepip # to install pip for Python3
@@ -347,7 +346,7 @@ Use the Cygwin installer to install the dependencies::
 
     python39 python39-devel
     python39-setuptools python39-pip python39-wheel python39-virtualenv
-    libssl-devel libxxhash-devel liblz4-devel libzstd-devel
+    libssl-devel libxxhash-devel liblz4-devel
     binutils gcc-g++ git make openssh
 
 Make sure to use a virtual environment to avoid confusions with any Python installed on Windows.

docs/usage/general/environment.rst.inc

@@ -218,9 +218,6 @@ Building:
     BORG_LIBLZ4_PREFIX
         Adds given prefix directory to the default locations. If a 'include/lz4.h' is found Borg
         will be linked against the system liblz4 instead of a bundled implementation. (setup.py)
-    BORG_LIBZSTD_PREFIX
-        Adds given prefix directory to the default locations. If a 'include/zstd.h' is found Borg
-        will be linked against the system libzstd instead of a bundled implementation. (setup.py)
 
 Please note:
 

pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
   "platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'",  # for others: 2.6+ works consistently.
   "argon2-cffi",
   "shtab>=1.8.0",
+  "backports-zstd; python_version < '3.14'", # for python < 3.14.
 ]
 
 [project.optional-dependencies]

scripts/Dockerfile.linux-run

@@ -10,7 +10,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libacl1-dev \
     libxxhash-dev \
     liblz4-dev \
-    libzstd-dev \
     libfuse3-dev \
     fuse3 \
     python3-dev \

scripts/msys2-install-deps

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,zstd,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
+pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
 
 if [ "$1" = "development" ]; then
 	pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist}

setup.py

@@ -160,7 +160,6 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
     compress_ext_kwargs = members_appended(
         dict(sources=[compress_source]),
         lib_ext_kwargs(pc, "BORG_LIBLZ4_PREFIX", "lz4", "liblz4", ">= 1.7.0"),
-        lib_ext_kwargs(pc, "BORG_LIBZSTD_PREFIX", "zstd", "libzstd", ">= 1.3.0"),
         dict(extra_compile_args=cflags),
     )
 

src/borg/compress.pyx

@@ -29,28 +29,21 @@ except ImportError:
 from .constants import MAX_DATA_SIZE, ROBJ_FILE_STREAM
 from .helpers import Buffer, DecompressionError
 
+import sys
+
+if sys.version_info >= (3, 14):
+    from compression import zstd
+else:
+    from backports import zstd
 
 
 cdef extern from "lz4.h":
     int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
     int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
     int LZ4_compressBound(int inputSize) nogil
 
-
-cdef extern from "zstd.h":
-    size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int  compressionLevel) nogil
-    size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t compressedSize) nogil
-    size_t ZSTD_compressBound(size_t srcSize) nogil
-    unsigned long long ZSTD_CONTENTSIZE_UNKNOWN
-    unsigned long long ZSTD_CONTENTSIZE_ERROR
-    unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) nogil
-    unsigned ZSTD_isError(size_t code) nogil
-    const char* ZSTD_getErrorName(size_t code) nogil
-
-
 buffer = Buffer(bytearray, size=0)
 
-
 cdef class CompressorBase:
     """
     base class for all (de)compression classes,
@@ -303,74 +296,32 @@ class LZMA(DecidingCompressor):
         except lzma.LZMAError as e:
             raise DecompressionError(str(e)) from None
 
-
 class ZSTD(DecidingCompressor):
-    """zstd compression / decompression (pypi: zstandard, gh: python-zstandard)"""
-    # This is a NOT THREAD SAFE implementation.
-    # Only ONE python context must be created at a time.
-    # It should work flawlessly as long as borg will call ONLY ONE compression job at time.
+    """
+    zstd compression / decompression (python stdlib (python >= 3.14))
+    """
     ID = 0x03
     name = 'zstd'
 
     def __init__(self, level=3, legacy_mode=False, **kwargs):
         super().__init__(level=level, legacy_mode=legacy_mode, **kwargs)
         self.level = level
 
-    def _decide(self, meta, idata):
-        """
-        Decides what to do with *data*. Returns (compressor, zstd_data).
-
-        *zstd_data* is the ZSTD result if *compressor* is ZSTD as well, otherwise it is None.
-        """
-        if not isinstance(idata, bytes):
-            idata = bytes(idata)  # code below does not work with memoryview
-        cdef int isize = len(idata)
-        cdef int osize
-        cdef char *source = idata
-        cdef char *dest
-        cdef int level = self.level
-        osize = ZSTD_compressBound(isize)
-        buf = buffer.get(osize)
-        dest = <char *> buf
-        with nogil:
-            osize = ZSTD_compress(dest, osize, source, isize, level)
-        if ZSTD_isError(osize):
-            raise Exception('zstd compress failed: %s' % ZSTD_getErrorName(osize))
-        # only compress if the result actually is smaller
-        if osize < isize:
-            return self, (meta, dest[:osize])
+    def _decide(self, meta, data):
+        zstd_data = zstd.compress(data, self.level)
+        if len(zstd_data) < len(data):
+            return self, (meta, zstd_data)
         else:
             return NONE_COMPRESSOR, (meta, None)
-
+    
     def decompress(self, meta, data):
-        meta, idata = super().decompress(meta, data)
-        if not isinstance(idata, bytes):
-            idata = bytes(idata)  # code below does not work with memoryview
-        cdef int isize = len(idata)
-        cdef unsigned long long osize
-        cdef unsigned long long rsize
-        cdef char *source = idata
-        cdef char *dest
-        osize = ZSTD_getFrameContentSize(source, isize)
-        if osize == ZSTD_CONTENTSIZE_ERROR:
-            raise DecompressionError('zstd get size failed: data was not compressed by zstd')
-        if osize == ZSTD_CONTENTSIZE_UNKNOWN:
-            raise DecompressionError('zstd get size failed: original size unknown')
+        meta, data = super().decompress(meta, data)
         try:
-            buf = buffer.get(osize)
-        except MemoryError:
-            raise DecompressionError('MemoryError')
-        dest = <char *> buf
-        with nogil:
-            rsize = ZSTD_decompress(dest, osize, source, isize)
-        if ZSTD_isError(rsize):
-            raise DecompressionError('zstd decompress failed: %s' % ZSTD_getErrorName(rsize))
-        if rsize != osize:
-            raise DecompressionError('zstd decompress failed: size mismatch')
-        data = dest[:osize]
-        self.check_fix_size(meta, data)
-        return meta, data
-
+            data = zstd.decompress(data)
+            self.check_fix_size(meta, data)
+            return meta, data
+        except zstd.ZstdError as e:
+            raise DecompressionError(str(e)) from None
 
 class ZLIB(DecidingCompressor):
     """