Build Nix derivations using Crane

.pre-commit-config.yaml

@@ -25,12 +25,6 @@ repos:
         language: system
         types: [python]
         require_serial: true
-      - id: biome
-        name: Biome formatting
-        entry: biome format --write --files-ignore-unknown=true --no-errors-on-unmatched
-        language: system
-        types: [json]
-        require_serial: true
       - id: rustfmt
         name: rustfmt
         entry: cargo fmt --

.semaphore/semaphore.yml

@@ -62,15 +62,15 @@ blocks:
       jobs:
         - name: "Build Opsqueue binary (in release mode) & run unit tests"
           commands:
-            - "just nix-build-bin | cachix push channable"
+            - "cachix watch-exec channable -- just nix-build-bin"
             # Once a day, we update the devenv cache.
             # Except when fresh, this takes < 1 second
             # but it cannot live in the prologue
             # as that might corrupt it
             - cache store nix-store-$(date -u -Idate) /nix
         - name: "Build Python library (in release mode)"
           commands:
-            - "just nix-build-python | cachix push channable"
+            - "cachix watch-exec channable -- just nix-build-python"
   - name: "Integration"
     dependencies:
       - "Build"

default.nix

@@ -4,7 +4,7 @@
 let
   pkgs = import ./nix/nixpkgs-pinned.nix { };
 
-  pythonEnv = pkgs.pythonChannable.withPackages (
+  pythonEnv = pkgs.python.withPackages (
     p: with p; [
       click
       mypy
@@ -22,9 +22,6 @@ let
     ]
   );
 
-  # We choose a minimal Rust channel to keep the Nix closure size smaller
-  rust = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml;
-
   defaultEnv = pkgs.buildEnv {
     name = "opsqueue-env-default";
     paths = [
@@ -39,7 +36,7 @@ let
       pkgs.ruff
 
       # For compiling the Rust parts
-      rust
+      pkgs.rustToolchain
       pkgs.sqlx-cli
 
       # Manage nix pins

justfile

@@ -47,11 +47,11 @@ test-integration *TEST_ARGS: build-bin build-python
   cd libs/opsqueue_python
   source "./.setup_local_venv.sh"
 
-  timeout 30 pytest --color=yes {{TEST_ARGS}}
+  pytest --color=yes {{TEST_ARGS}}
 
 # Python integration test suite, using artefacts built through Nix. Args are forwarded to pytest
 [group('nix')]
-nix-test-integration *TEST_ARGS:
+nix-test-integration *TEST_ARGS: nix-build
   #!/usr/bin/env bash
   set -euxo pipefail
   nix_build_python_library_dir=$(just nix-build-python)
@@ -61,7 +61,7 @@ nix-test-integration *TEST_ARGS:
   export OPSQUEUE_VIA_NIX=true
   export RUST_LOG="opsqueue=debug"
 
-  timeout 30 pytest --color=yes {{TEST_ARGS}}
+  pytest --color=yes {{TEST_ARGS}}
 
 # Run all linters, fast and slow
 [group('lint')]
@@ -89,15 +89,15 @@ mypy:
 
 # Build Nix-derivations of binary and all libraries (release profile)
 [group('nix')]
-nix-build: (_nix-build "opsqueue" "pythonChannable.pkgs.opsqueue_python")
+nix-build: (_nix-build "opsqueue" "python.pkgs.opsqueue_python")
 
 # Build Nix-derivation of binary (release profile)
 [group('nix')]
 nix-build-bin: (_nix-build "opsqueue")
 
 # Build Nix-derivation of Python client library (release profile)
 [group('nix')]
-nix-build-python: (_nix-build "pythonChannable.pkgs.opsqueue_python")
+nix-build-python: (_nix-build "python.pkgs.opsqueue_python")
 
 _nix-build +TARGETS:
   nix build --file nix/nixpkgs-pinned.nix --print-out-paths --print-build-logs --no-link --option sandbox true {{TARGETS}}

libs/opsqueue_python/examples/tracing/tracing_consumer.py

@@ -8,7 +8,7 @@
     # ConsoleSpanExporter,
 )
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
-from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource  # type: ignore[attr-defined]
 
 logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
 

libs/opsqueue_python/examples/tracing/tracing_producer.py

@@ -13,7 +13,7 @@
     # ConsoleSpanExporter,
 )
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
-from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource  # type: ignore[attr-defined]
 
 from opsqueue.producer import ProducerClient
 
@@ -75,7 +75,7 @@ def added_baggage(
         yield
     finally:
         for attached_token in attached_context_tokens:
-            opentelemetry.context.detach(attached_token)
+            opentelemetry.context.detach(attached_token)  # type: ignore[arg-type]
 
 
 if __name__ == "__main__":

libs/opsqueue_python/opsqueue_python.nix

@@ -1,62 +1,92 @@
 {
+  # Builtin
+  pkgs,
   lib,
+
+  # Rust version. (Override this with an overlay if you like)
+  rustToolchain,
+
+  # Native build dependencies:
+  python,
+  maturin,
   buildPythonPackage,
-  rustPlatform,
   perl,
   git,
-  # Python packages:
+
+  # Python package dependencies:
   cbor2,
   opentelemetry-api,
   opentelemetry-exporter-otlp,
   opentelemetry-sdk,
 }:
 let
-  root = ../../.;
-  util = import (root + /nix/util.nix) { inherit lib; };
-in
-buildPythonPackage rec {
-  pname = "opsqueue";
-  version = "0.1.0";
-  pyproject = true;
+  sources = import ../../nix/sources.nix;
+  crane = import sources.crane { pkgs = pkgs; };
+  craneLib = crane.overrideToolchain (pkgs: rustToolchain);
+  extraFileFilter = path: _type: builtins.match ".*(db|sql|py|md)$" path != null;
+  fileFilter = path: type: (extraFileFilter path type) || (craneLib.filterCargoSources path type);
 
-  src = util.fileFilter {
-    name = "opsqueue_python";
-    src = root;
+  src = lib.cleanSourceWith {
+    src = ../../.;
+    name = "opsqueue";
+    filter = fileFilter;
+  };
 
-    # We're copying slightly too much to the Nix store here,
-    # but using the more granular file filter was very error-prone.
-    # This is one thing that could be improved a little in the future.
-    srcGlobalWhitelist = [
-      ".py"
-      ".pyi"
-      "py.typed"
-      ".rs"
-      ".toml"
-      ".lock"
-      ".db"
-      ".md"
-    ];
+  crateName = craneLib.crateNameFromCargoToml { cargoToml = ./Cargo.toml; };
+  pname = crateName.pname;
+  version = crateName.version;
+  commonArgs = {
+    inherit src version pname;
+    strictDeps = true;
+    nativeBuildInputs = [ python ];
+    cargoExtraArgs = "--package opsqueue_python";
   };
+  cargoArtifacts = craneLib.buildDepsOnly (commonArgs // { pname = pname; });
 
-  cargoDeps = rustPlatform.importCargoLock { lockFile = ../../Cargo.lock; };
+  wheelTail = "py3-abi3-linux_x86_64";
+  wheelName = "opsqueue-${version}-${wheelTail}.whl";
 
-  env = {
-    DATABASE_URL = "sqlite://./opsqueue/opsqueue_example_database_schema.db";
-  };
+  crateWheel =
+    (craneLib.buildPackage (commonArgs // { inherit cargoArtifacts; })).overrideAttrs
+      (old: {
+        nativeBuildInputs = old.nativeBuildInputs ++ [ maturin ];
 
-  pythonImportsCheck = [ pname ];
+        # We intentionally _override_ rather than extend the buildPhase
+        # as Maturin itself calls `cargo build`, no need to call it twice.
+        buildPhase = ''
+          cargo --version
+          maturin build --release --offline --target-dir ./target --manifest-path "./libs/opsqueue_python/Cargo.toml"
+        '';
 
-  maturinBuildFlags = [
-    "--manifest-path"
-    "./libs/opsqueue_python/Cargo.toml"
-  ];
+        # We build a single wheel
+        # but by convention its name is based on the precise combination of
+        # Python version + OS version + architecture + ...
+        #
+        # The Nix hash already covers us for uniqueness and compatibility.
+        # So this 'trick' copies it to a predictably named file.
+        #
+        # Just like `buildPhase`, we override rather than extend
+        # because we are only interested in the wheel output of Maturin as a whole.
+        # (which is an archive inside of it containing the `.so` cargo built)
+        installPhase = ''
+          mkdir -p $out
+          for wheel in ./target/wheels/*.whl ; do
+            cp "$wheel" $out/${wheelName}
+          done
+        '';
 
-  nativeBuildInputs = with rustPlatform; [
-    perl
-    git
-    cargoSetupHook
-    maturinBuildHook
-  ];
+        # There are no Rust unit tests in the Python FFI library currently,
+        # so we can skip rebuilding opsqueue_python for tests.
+        doCheck = false;
+      });
+in
+buildPythonPackage {
+  pname = pname;
+  format = "wheel";
+  version = crateName.version;
+  src = "${crateWheel}/${wheelName}";
+  doCheck = false;
+  pythonImportsCheck = [ "opsqueue" ];
 
   propagatedBuildInputs = [
     cbor2

libs/opsqueue_python/pyproject.toml

@@ -52,15 +52,15 @@ test = [
     "pytest==8.3.3",
     "pytest-random-order==1.1.1",
     "pytest-parallel==0.1.1",
-    "pytest-timeout==2.4.0",
+    # "pytest-timeout==2.4.0",
     "py==1.11.0", # Needs to be manually specified because of this issue: https://github.com/kevlened/pytest-parallel/issues/118
 ]
 
 [tool.pytest.ini_options]
 # We ensure tests never rely on global state,
 # by running them in a random order, and in parallel:
-addopts = "--random-order --workers=auto"
-# Individual tests should be very fast. They should never take multiple seconds
-# If after 20sec (accomodating for a toaster-like PC) there is no progress,
-# assume a deadlock
-timeout=20
+addopts = "--random-order --workers=4"
+# # Individual tests should be very fast. They should never take multiple seconds
+# # If after 20sec (accomodating for a toaster-like PC) there is no progress,
+# # assume a deadlock
+# timeout=20

libs/opsqueue_python/src/common.rs

@@ -428,11 +428,8 @@ pub async fn check_signals_in_background() -> FatalPythonException {
             if let Err(err) = py.check_signals() {
                 // A signal was triggered
                 Some(err)
-            } else if let Some(err) = PyErr::take(py) {
-                // A non-signal Python exception was thrown
-                return Some(err);
             } else {
-                return None;
+                PyErr::take(py)
             }
         });
         if let Some(res) = res {

libs/opsqueue_python/src/consumer.rs

@@ -418,7 +418,7 @@ impl ConsumerClient {
             }
 
             done_count = done_count.saturating_add(1);
-            if done_count % 50 == 0 {
+            if done_count.is_multiple_of(50) {
                 tracing::info!("Processed {} chunks", done_count);
             }
         }

nix/nixpkgs-pinned.nix

@@ -10,7 +10,8 @@ let
   used_overlays = [
     (import sources.rust-overlay)
     (import ./overlay.nix)
-  ] ++ overlays;
+  ]
+  ++ overlays;
 
   nixpkgsArgs = args // {
     overlays = used_overlays;

nix/overlay.nix

@@ -1,16 +1,25 @@
 # Overlay for Nixpkgs which holds all opsqueue related packages.
 final: prev:
 let
+  sources = import ./sources.nix;
   pythonOverlay = import ./python-overlay.nix;
+
+  # We want to use the same Rust version in Nix
+  # as we use when _not_ using Nix.
+  #
+  # When using opsqueue as part of your own Nix derivations,
+  # be sure to use an overlay to set `rustToolchain`
+  # to the desired Rust version
+  # if you want to use a different version from the default.
+  rustToolchain = final.rust-bin.fromRustupToolchainFile ../rust-toolchain.toml;
+
+  crane = import sources.crane { pkgs = final; };
+  craneLib = crane.overrideToolchain (pkgs: rustToolchain);
 in
 {
+  inherit rustToolchain;
   opsqueue = final.callPackage ../opsqueue/opsqueue.nix { };
 
-  # The explicit choice is made not to override `python312`, as this will cause a rebuild of many
-  # packages when nixpkgs uses python 3.12 as default python environment.
-  # These packages should not be affected, e.g. cachix. This is because of a transitive
-  # dependency on the Python packages that we override.
-  # In our case cachix > ghc > shpinx > Python libraries.
-  pythonChannable = prev.python312.override { packageOverrides = pythonOverlay; };
+  python = prev.python312.override { packageOverrides = pythonOverlay; };
 
 }

nix/sources.json

@@ -1,14 +1,26 @@
 {
+    "crane": {
+        "branch": "master",
+        "description": "A Nix library for building cargo projects. Never build twice thanks to incremental artifact caching.",
+        "homepage": "https://crane.dev",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "0bda7e7d005ccb5522a76d11ccfbf562b71953ca",
+        "sha256": "1ndhiw867qiwr3kswm2mb81y3xcfdwz92xvs00r6jd4blxsv7z09",
+        "type": "tarball",
+        "url": "https://github.com/ipetkov/crane/archive/0bda7e7d005ccb5522a76d11ccfbf562b71953ca.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
     "nixpkgs": {
         "branch": "nixpkgs-unstable",
         "description": "Nix Packages collection",
         "homepage": "",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d19cf9dfc633816a437204555afeb9e722386b76",
-        "sha256": "1wirhlw7cqaypgakyfz9ikv7nxdq3il0fk38cdrdmps2zn1l4ccp",
+        "rev": "ed142ab1b3a092c4d149245d0c4126a5d7ea00b0",
+        "sha256": "1h7v295lpjfxpxkag2csam7whx918sdypixdi8i85vlb707gg0vm",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/d19cf9dfc633816a437204555afeb9e722386b76.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/ed142ab1b3a092c4d149245d0c4126a5d7ea00b0.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "rust-overlay": {