From 592e9a944b8e4d9eb3fd228da8238dccf9bcef00 Mon Sep 17 00:00:00 2001 From: "w.pomp" Date: Thu, 29 Oct 2026 12:00:24 +0100 Subject: [PATCH] - pypi wheels against the libczirw-sys submodule (self-contained Eigen build, msvc runtime alignment, bindgen flags) - and slim down publish_pypi.yml: no pinned zig/clang versions, no cmake shim, no bindgen env vars; only the ring /imsvc clang shim remains --- .github/workflows/publish_pypi.yml | 151 ++++++++++++----------------- .gitmodules | 3 + Cargo.toml | 7 ++ libczirw-sys | 1 + 4 files changed, 72 insertions(+), 90 deletions(-) create mode 160000 libczirw-sys diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 37449b0..05706b6 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -33,22 +33,18 @@ jobs: ~/.osxcross key: cache-ubuntu-maturin-cross-compile - - name: Install llvm, cmake and clang + - name: Install build dependencies run: | - if ! command -v llvm-dlltool >/dev/null 2>&1; then - sudo apt update - sudo apt install -y llvm cmake build-essential libclang-dev clang - fi - # the stock clang on noble is 18, but the 2025+ MSVC STL bundled with - # cargo-xwin requires clang >= 19 (yvals_core.h "error STL1000: expected - # Clang 19.0.0 or newer"). Install a newer toolchain from apt.llvm.org so - # its clang-cl driver, lld-link and libclang (bindgen) are all new enough. - if ! command -v clang-24 >/dev/null 2>&1; then - echo "deb [trusted=yes] https://apt.llvm.org/noble/ llvm-toolchain-noble main" | sudo tee /etc/apt/sources.list.d/llvm-toolchain-noble.list - sudo apt update - # libeigen3-dev: Eigen 3.4.0 for libczirw (see the cmake-shim step below) - sudo apt install -y clang-24 lld-24 libclang-24-dev libeigen3-dev - fi + # the stock clang on noble (18) is too old for the 2025+ MSVC STL that + # cargo-xwin downloads (yvals_core.h: "error STL1000: expected Clang + # 19.0.0 or newer"), so take the newest clang / lld / libclang / llvm + # tools from apt.llvm.org (generic package names, nothing pinned) + sudo apt update + echo "deb [trusted=yes] https://apt.llvm.org/noble/ llvm-toolchain-noble main" | sudo tee /etc/apt/sources.list.d/llvm-toolchain-noble.list >/dev/null + sudo apt update + sudo apt install -y build-essential cmake clang lld libclang-dev llvm + clang --version | head -1 + command -v clang clang-cl lld-link llvm-lib shell: bash - name: Install Rust @@ -65,12 +61,7 @@ jobs: - name: Install sccache and maturin run: | - python -m pip install --upgrade pip - # ziglang >= 0.17 ships a bundled libc++ that requires -D_LIBCPP_* config - # macros only cargo-zigbuild >= 0.24.0 passes to bindgen; otherwise - # libczirw-sys fails with "hardening.h ... _LIBCPP_ASSERTION_SEMANTIC_DEFAULT - # is not defined". Pin the combo that is known to work. - python -m pip install --upgrade maturin "ziglang==0.17.0" "cargo-zigbuild==0.24.0" + python -m pip install --upgrade pip maturin ziglang cargo-zigbuild # in case an older copy was cargo-installed into the CARGO_HOME cache, # prevent it from shadowing the pip-installed one rm -f "${CARGO_HOME:?}/bin/cargo-zigbuild" || true @@ -95,83 +86,71 @@ jobs: tar -xzf osxcross.tar.gz -C ~/ mv ~/osxcross ~/.osxcross fi + shell: bash - - name: Tune the libczirw-sys cmake build (system Eigen, dynamic CRT) + - name: Shim clang for ring's aarch64-windows build run: | - # libczirw-sys cmake build clones Eigen3 from gitlab.com (ExternalProject); - # that clone is flaky on this runner (it is the only git-based network step - # left in the build). noble libeigen3-dev is the same Eigen 3.4.0 that libczi - # pins, and libczi has an upstream option to use it - flip that option for - # the libCZI project via CMAKE_PROJECT_INCLUDE. cmake does not read that - # variable from the environment, so a cmake shim injects it (-D); the shim is - # scoped to invocations that look like the libczi configure (no-op for - # --build / --version / other uses). - # the override also aligns the msvc CRT: libCZI forces the static runtime - # (/MT) while the rest of the build (Rust std, cc-compiled vendored libs, - # /defaultlib:msvcrt) is dynamic (/MD), and lld-link rejects the mix with - # /failifmismatch:RuntimeLibrary. no effect on the non-windows builds. + # ring 0.17.14 forces a plain `clang` for its C build on + # aarch64-pc-windows-msvc (its "must use Clang on Win AArch64" + # workaround), while the CFLAGS injected by cargo-xwin carry + # clang-cl-style `/imsvc ` pairs that plain clang rejects. + # This shim rewrites them to `-I ` and execs the real clang + # found further down PATH; all other invocations pass through + # untouched, and tools with other names (clang-cl, lld-link) are not + # affected. mkdir -p /root/ci-shims - cat > /root/cm-override-libczi.cmake <<'CMAKEEOF' - if (CMAKE_PROJECT_NAME STREQUAL "libCZI") - set(LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_EIGEN3 ON CACHE BOOL "" FORCE) - set(LIBCZI_DO_NOT_SET_MSVC_RUNTIME_LIBRARY ON CACHE BOOL "" FORCE) - set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDLL$<$:Debug>" CACHE STRING "" FORCE) - endif() - CMAKEEOF - cat > /root/ci-shims/cmake <<'SHIMEOF' + cat > /root/ci-shims/clang <<'EOF' #!/usr/bin/env bash - case "$*" in - --build*) - exec /usr/bin/cmake "$@" ;; - *libczi*) - exec /usr/bin/cmake -DCMAKE_PROJECT_INCLUDE=/root/cm-override-libczi.cmake "$@" ;; - *) - exec /usr/bin/cmake "$@" ;; - esac - SHIMEOF - chmod +x /root/ci-shims/cmake - echo "--- /root/cm-override-libczi.cmake ---" - cat /root/cm-override-libczi.cmake - echo "--- /root/ci-shims/cmake ---" - cat /root/ci-shims/cmake + args=() + prev_imsvc=0 + for a in "$@"; do + if [ "$prev_imsvc" = "1" ]; then + args+=("-I$a") + prev_imsvc=0 + continue + fi + case "$a" in + /imsvc) prev_imsvc=1 ;; + /imsvc:*) args+=("-I${a#/imsvc:}") ;; + *) args+=("$a") ;; + esac + done + self_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + real="" + IFS=":" read -r -a _dirs <<< "$PATH" + IFS="" + for d in "${_dirs[@]}"; do + [ "$d" = "$self_dir" ] && continue + if [ -x "$d/clang" ]; then real="$d/clang"; break; fi + done + [ -n "$real" ] || real=/usr/bin/clang + exec "$real" "${args[@]}" + EOF + chmod +x /root/ci-shims/clang shell: bash - name: Build wheels - env: - # ziglang >= 0.17's bundled libc++ needs these -D_LIBCPP_* macros when - # parsed by bindgen/libclang; cargo-zigbuild >= 0.24.0 supplies them, but - # the runner may still resolve an older 0.23.1 — pre-set them so any - # cargo-zigbuild version works (it appends its own flags to these) - BINDGEN_EXTRA_CLANG_ARGS_x86_64-apple-darwin: "-D_LIBCPP_ASSERTION_SEMANTIC_DEFAULT=_LIBCPP_ASSERTION_SEMANTIC_ENFORCE -D_LIBCPP_PSTL_BACKEND_SERIAL -D_LIBCPP_HAS_VENDOR_AVAILABILITY_ANNOTATIONS=0 -D_LIBCPP_HAS_TERMINAL -D_LIBCPP_HAS_RANDOM_DEVICE -D_LIBCPP_HAS_NO_STD_MODULES -D_LIBCPP_HAS_FILESYSTEM=1" - BINDGEN_EXTRA_CLANG_ARGS_aarch64-apple-darwin: "-D_LIBCPP_ASSERTION_SEMANTIC_DEFAULT=_LIBCPP_ASSERTION_SEMANTIC_ENFORCE -D_LIBCPP_PSTL_BACKEND_SERIAL -D_LIBCPP_HAS_VENDOR_AVAILABILITY_ANNOTATIONS=0 -D_LIBCPP_HAS_TERMINAL -D_LIBCPP_HAS_RANDOM_DEVICE -D_LIBCPP_HAS_NO_STD_MODULES -D_LIBCPP_HAS_FILESYSTEM=1" - BINDGEN_EXTRA_CLANG_ARGS_x86_64-unknown-linux-gnu: "-D_LIBCPP_ASSERTION_SEMANTIC_DEFAULT=_LIBCPP_ASSERTION_SEMANTIC_ENFORCE -D_LIBCPP_PSTL_BACKEND_SERIAL -D_LIBCPP_HAS_VENDOR_AVAILABILITY_ANNOTATIONS=0 -D_LIBCPP_HAS_TERMINAL -D_LIBCPP_HAS_RANDOM_DEVICE -D_LIBCPP_HAS_NO_STD_MODULES -D_LIBCPP_HAS_FILESYSTEM=1" - BINDGEN_EXTRA_CLANG_ARGS_aarch64-unknown-linux-gnu: "-D_LIBCPP_ASSERTION_SEMANTIC_DEFAULT=_LIBCPP_ASSERTION_SEMANTIC_ENFORCE -D_LIBCPP_PSTL_BACKEND_SERIAL -D_LIBCPP_HAS_VENDOR_AVAILABILITY_ANNOTATIONS=0 -D_LIBCPP_HAS_TERMINAL -D_LIBCPP_HAS_RANDOM_DEVICE -D_LIBCPP_HAS_NO_STD_MODULES -D_LIBCPP_HAS_FILESYSTEM=1" - BINDGEN_EXTRA_CLANG_ARGS_x86_64-unknown-linux-musl: "-D_LIBCPP_ASSERTION_SEMANTIC_DEFAULT=_LIBCPP_ASSERTION_SEMANTIC_ENFORCE -D_LIBCPP_PSTL_BACKEND_SERIAL -D_LIBCPP_HAS_VENDOR_AVAILABILITY_ANNOTATIONS=0 -D_LIBCPP_HAS_TERMINAL -D_LIBCPP_HAS_RANDOM_DEVICE -D_LIBCPP_HAS_NO_STD_MODULES -D_LIBCPP_HAS_FILESYSTEM=1" - BINDGEN_EXTRA_CLANG_ARGS_aarch64-unknown-linux-musl: "-D_LIBCPP_ASSERTION_SEMANTIC_DEFAULT=_LIBCPP_ASSERTION_SEMANTIC_ENFORCE -D_LIBCPP_PSTL_BACKEND_SERIAL -D_LIBCPP_HAS_VENDOR_AVAILABILITY_ANNOTATIONS=0 -D_LIBCPP_HAS_TERMINAL -D_LIBCPP_HAS_RANDOM_DEVICE -D_LIBCPP_HAS_NO_STD_MODULES -D_LIBCPP_HAS_FILESYSTEM=1" shell: bash run: | export PATH="$HOME/.osxcross/bin:$PATH" - # put the cmake shim (system-Eigen override for libczirw) in front - export PATH="/root/ci-shims:$PATH" - command -v cmake - echo "HOME=$HOME CARGO_HOME=$CARGO_HOME RUSTUP_HOME=$RUSTUP_HOME" - command -v cargo-zigbuild || echo "cargo-zigbuild: not on PATH" - cargo-zigbuild --version 2>&1 || true - command -v rustup - command -v cargo - rustup show - cargo --version + # the shim (for ring's forced plain `clang`) goes in front of the + # llvm tools so clang-cl / lld-link (distinct names) still resolve to + # the real toolchain; bindgen finds libclang via LIBCLANG_PATH + LLVM_BIN=$(dirname "$(readlink -f "$(command -v clang)")") + export PATH="/root/ci-shims:$LLVM_BIN:$PATH" + export LIBCLANG_PATH="$(dirname "$LLVM_BIN")/lib" rm -rf dist maturin sdist --out dist -m ndbioimage/Cargo.toml - + rustup target add x86_64-apple-darwin maturin build --release --out dist --target x86_64-apple-darwin --zig --auditwheel repair -m ndbioimage/Cargo.toml rustup target add aarch64-apple-darwin maturin build --release --out dist --target aarch64-apple-darwin --zig --auditwheel repair -m ndbioimage/Cargo.toml - + rustup target add x86_64-unknown-linux-gnu - # the x86_64-gnu build links the runner's host liblzma (lzma-sys), which needs - # glibc symbols up to 2.34 - higher than the default manylinux_2_17 baseline + # the x86_64-gnu build links the runner's host liblzma (lzma-sys), + # which needs glibc symbols up to 2.34 - higher than the + # manylinux_2_17 baseline maturin build --release --out dist --target x86_64-unknown-linux-gnu --zig --auditwheel repair --compatibility manylinux_2_34 -m ndbioimage/Cargo.toml rustup target add aarch64-unknown-linux-gnu maturin build --release --out dist --target aarch64-unknown-linux-gnu --zig --auditwheel repair --compatibility manylinux_2_34 -m ndbioimage/Cargo.toml @@ -179,15 +158,7 @@ jobs: maturin build --release --out dist --target x86_64-unknown-linux-musl --zig --auditwheel repair -m ndbioimage/Cargo.toml rustup target add aarch64-unknown-linux-musl maturin build --release --out dist --target aarch64-unknown-linux-musl --zig --auditwheel repair -m ndbioimage/Cargo.toml - - # cargo-xwin resolves clang-cl / lld-link through PATH and bindgen finds - # libclang via LIBCLANG_PATH, so prefer the >=19 toolchain over the stock - # clang-18 (rejected by the 2025+ MSVC STL with STL1000) for the msvc builds - export PATH="/usr/lib/llvm-24/bin:$PATH" - export LIBCLANG_PATH="/usr/lib/llvm-24/lib" - command -v clang-cl - clang-cl --version - command -v lld-link + rustup target add x86_64-pc-windows-msvc maturin build --release --out dist --target x86_64-pc-windows-msvc --auditwheel repair -m ndbioimage/Cargo.toml rustup target add aarch64-pc-windows-msvc diff --git a/.gitmodules b/.gitmodules index 6fb5dd1..a425167 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "image-registration"] path = image-registration url = https://git.pomppervova.nl/wim/image-registration.git +[submodule "libczirw-sys"] + path = libczirw-sys + url = https://git.pomppervova.nl/wim/libczirw-sys.git diff --git a/Cargo.toml b/Cargo.toml index ef2dbe6..eaca1a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,3 +23,10 @@ debug = true [profile.dev.package.backtrace] opt-level = 3 + +# Build against the libczirw-sys checkout in the submodule of that name: +# its fixes (self-contained cmake build via vendored Eigen, msvc C runtime +# alignment for xwin, bindgen flags for zig's libc++) are not yet published +# to crates.io; drop this patch once a release containing them is available +[patch.crates-io] +libczirw-sys = { path = "libczirw-sys" } diff --git a/libczirw-sys b/libczirw-sys new file mode 160000 index 0000000..d6fe6ec --- /dev/null +++ b/libczirw-sys @@ -0,0 +1 @@ +Subproject commit d6fe6ec73368b6940bb64d004082ef1171fff26c