From e55152c55d051631e59b7762596a0b7474fc05cd Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Sat, 3 Oct 2026 19:35:01 +0200 Subject: [PATCH] - use a fork of the tiff crate to open RGBPalette tiffs --- .cargo/config.toml | 2 - .gitignore | 6 +- .gitmodules | 6 + Cargo.toml | 82 ++-------- image-registration | 1 + image-tiff | 1 + ndbioimage/Cargo.toml | 82 ++++++++++ README.md => ndbioimage/README.md | 0 build.rs => ndbioimage/build.rs | 24 ++- {py => ndbioimage/py}/ndbioimage/__init__.py | 32 ++-- .../py}/ndbioimage/ndbioimage_rs/__init__.pyi | 23 ++- .../ndbioimage/ndbioimage_rs/transforms.pyi | 16 ++ {py => ndbioimage/py}/ndbioimage/py.typed | 0 .../py}/ndbioimage/transforms.pyi | 0 pyproject.toml => ndbioimage/pyproject.toml | 0 {src => ndbioimage/src}/axes.rs | 0 {src => ndbioimage/src}/cache.rs | 107 +++++++++++++ {src => ndbioimage/src}/colors.rs | 0 {src => ndbioimage/src}/error.rs | 9 +- {src => ndbioimage/src}/lib.rs | 0 {src => ndbioimage/src}/main.rs | 0 {src => ndbioimage/src}/metadata.rs | 0 {src => ndbioimage/src}/movie.rs | 0 {src => ndbioimage/src}/py.rs | 2 +- {src => ndbioimage/src}/py/imread.rs | 28 +++- {src => ndbioimage/src}/py/transforms.rs | 100 +++++++++++- {src => ndbioimage/src}/readers.rs | 0 .../src}/readers/bioformats_java.rs | 146 ++++++++++++++++-- .../src}/readers/bioformats_rust.rs | 0 {src => ndbioimage/src}/readers/czi.rs | 0 {src => ndbioimage/src}/readers/tiff.rs | 61 ++++++-- {src => ndbioimage/src}/readers/tiffseq.rs | 31 ++-- {src => ndbioimage/src}/stats.rs | 0 {src => ndbioimage/src}/tiffwrite.rs | 40 +++++ {src => ndbioimage/src}/transforms.rs | 41 ++++- {src => ndbioimage/src}/utils.rs | 0 {src => ndbioimage/src}/view.rs | 32 +++- {tests => ndbioimage/tests}/test_open.py | 0 {tests => ndbioimage/tests}/test_slicing.py | 0 {tests => ndbioimage/tests}/test_ufuncs.py | 0 py/ndbioimage/__init__.pyi | 3 - 41 files changed, 728 insertions(+), 147 deletions(-) delete mode 100644 .cargo/config.toml create mode 100644 .gitmodules create mode 160000 image-registration create mode 160000 image-tiff create mode 100644 ndbioimage/Cargo.toml rename README.md => ndbioimage/README.md (100%) rename build.rs => ndbioimage/build.rs (84%) rename {py => ndbioimage/py}/ndbioimage/__init__.py (83%) rename {py => ndbioimage/py}/ndbioimage/ndbioimage_rs/__init__.pyi (96%) rename {py => ndbioimage/py}/ndbioimage/ndbioimage_rs/transforms.pyi (94%) rename {py => ndbioimage/py}/ndbioimage/py.typed (100%) rename {py => ndbioimage/py}/ndbioimage/transforms.pyi (100%) rename pyproject.toml => ndbioimage/pyproject.toml (100%) rename {src => ndbioimage/src}/axes.rs (100%) rename {src => ndbioimage/src}/cache.rs (72%) rename {src => ndbioimage/src}/colors.rs (100%) rename {src => ndbioimage/src}/error.rs (94%) rename {src => ndbioimage/src}/lib.rs (100%) rename {src => ndbioimage/src}/main.rs (100%) rename {src => ndbioimage/src}/metadata.rs (100%) rename {src => ndbioimage/src}/movie.rs (100%) rename {src => ndbioimage/src}/py.rs (98%) rename {src => ndbioimage/src}/py/imread.rs (98%) rename {src => ndbioimage/src}/py/transforms.rs (89%) rename {src => ndbioimage/src}/readers.rs (100%) rename {src => ndbioimage/src}/readers/bioformats_java.rs (74%) rename {src => ndbioimage/src}/readers/bioformats_rust.rs (100%) rename {src => ndbioimage/src}/readers/czi.rs (100%) rename {src => ndbioimage/src}/readers/tiff.rs (86%) rename {src => ndbioimage/src}/readers/tiffseq.rs (95%) rename {src => ndbioimage/src}/stats.rs (100%) rename {src => ndbioimage/src}/tiffwrite.rs (88%) rename {src => ndbioimage/src}/transforms.rs (90%) rename {src => ndbioimage/src}/utils.rs (100%) rename {src => ndbioimage/src}/view.rs (98%) rename {tests => ndbioimage/tests}/test_open.py (100%) rename {tests => ndbioimage/tests}/test_slicing.py (100%) rename {tests => ndbioimage/tests}/test_ufuncs.py (100%) delete mode 100644 py/ndbioimage/__init__.pyi diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index 8b1d260..0000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[patch.crates-io] -tiff = { git = "https://git.pomppervova.nl/wim/image-tiff.git" } diff --git a/.gitignore b/.gitignore index 75e4da8..8414cb7 100644 --- a/.gitignore +++ b/.gitignore @@ -72,11 +72,11 @@ docs/_build/ # Pyenv .python-version -/tests/files/* +/ndbioimage/tests/files/* AGENTS.md .agentbridge .agent-work -py/ndbioimage/jassets -py/ndbioimage/deps +/ndbioimage/py/ndbioimage/jassets +/ndbioimage/py/ndbioimage/deps /build_wheels.sh diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..6fb5dd1 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "image-tiff"] + path = image-tiff + url = https://git.pomppervova.nl/wim/image-tiff.git +[submodule "image-registration"] + path = image-registration + url = https://git.pomppervova.nl/wim/image-registration.git diff --git a/Cargo.toml b/Cargo.toml index 341823e..58df30b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,11 @@ -[package] -name = "ndbioimage" +[workspace] +resolver = "3" +members = ["ndbioimage", "image-tiff", "image-registration"] + +[workspace.package] version = "0.2.0" edition = "2024" +# the workspace must satisfy every member, so this is the highest `rust-version` of the members rust-version = "1.94.0" authors = ["Wim Pomp "] license = "MIT OR Apache-2.0" @@ -9,77 +13,13 @@ description = "Read bio image formats using the bio-formats java package." homepage = "https://git.pomppervova.nl/wim/ndbioimage/src/branch/rs" repository = "https://git.pomppervova.nl/wim/ndbioimage/src/branch/rs" documentation = "https://docs.rs/ndbioimage" -readme = "README.md" +# inherited paths are resolved relative to the workspace root, not the member, so this has to point +# at the member's readme +readme = "ndbioimage/README.md" keywords = ["bioformats", "imread", "ndarray", "metadata"] categories = ["multimedia::images", "science"] -exclude = ["/tests"] - -[lib] -name = "ndbioimage" -crate-type = ["cdylib", "rlib"] - -[dependencies] -bioformats = { version = "0.1", optional = true, default-features = false, features = ["zarr", "tissuefaxs"] } -clap = { version = "4", features = ["derive"] } -console = { version = "0.16", optional = true } -downloader = { version = "0.2", optional = true, default-features = false, features = ["rustls-tls"] } -ffmpeg-sidecar = { version = "2", optional = true } -itertools = "0.15" -image-registration = { path = "../image-registration", optional = true } -indexmap = { version = "2", features = ["serde"] } -indicatif = { version = "0.18", features = ["rayon"], optional = true } -j4rs = { version = "0.25", optional = true } -libczirw-sys = { version = "0.5", optional = true } -ndarray = { version = "0.17", features = ["serde"] } -num = "0.4" -numpy = { version = "0.29", optional = true } -ome-metadata = "0.5" -ordered-float = { version = "5", optional = true } -phf = { version = "0.14", features = ["macros"] } -postcard = { version = "1", features = ["use-std"], optional = true } -pyo3 = { version = "0.29", features = ["abi3-py310", "anyhow", "generate-import-lib"], optional = true } -pyo3-stub-gen = { version = "0.23", optional = true } -rayon = { version = "1", optional = true } -regex = "1" -serde = { version = "1", features = ["rc", "derive"] } -serde_with = "3" -strum = { version = "0.28", features = ["derive"] } -thiserror = "2" -tiff = { version = "0.11", features = ["zstd"], optional = true } -tiffwrite = { version = "2026.6.0", optional = true } -tokio = { version = "1", features = ["rt", "rt-multi-thread"], optional = true } -thread_local = { version = "1", optional = true } -xmltree = { version = "0.12", optional = true } -yaml_serde = { version = "0.10", optional = true } - -[dev-dependencies] -rayon = "1" -regex = "1" - -[build-dependencies] -j4rs = "0.25" -ffmpeg-sidecar = "2" -retry = "2" -toml = "1" - -[features] -default = ["bioformats_java", "gpl-formats", "czi", "tiff", "tiffseq", "movie", "tiffwrite"] -all = ["bioformats_java", "bioformats_rust", "czi", "gpl-formats", "movie", "tiffseq", "tiffwrite", "tiff", "transforms"] -gpl-formats = [] -python = ["dep:pyo3", "dep:numpy", "dep:pyo3-stub-gen", "dep:postcard", "ome-metadata/python"] -czi = ["dep:libczirw-sys", "dep:xmltree", "dep:thread_local"] -bioformats_rust = ["dep:bioformats", "dep:thread_local"] -bioformats_java = ["dep:j4rs", "dep:thread_local", "dep:downloader"] -tiffwrite = ["dep:tiffwrite", "dep:indicatif", "dep:console", "dep:rayon"] -tiffseq = ["dep:tiff", "dep:yaml_serde"] -tiff = ["dep:tiff", "dep:thread_local"] -movie = ["dep:ffmpeg-sidecar", "dep:tokio", "dep:ordered-float", "dep:indicatif", "dep:console"] -transforms = ["dep:image-registration", "dep:yaml_serde"] - -[package.metadata.docs.rs] -no-default-features = true -features = ["bioformats_java", "bioformats_rust", "czi", "movie", "tiffseq", "tiffwrite", "tiff", "transforms"] +# profiles are only honoured in the workspace root manifest [profile.test] inherits = "release" @@ -87,4 +27,4 @@ inherits = "release" debug = true [profile.dev.package.backtrace] -opt-level = 3 \ No newline at end of file +opt-level = 3 diff --git a/image-registration b/image-registration new file mode 160000 index 0000000..7eb39a2 --- /dev/null +++ b/image-registration @@ -0,0 +1 @@ +Subproject commit 7eb39a2fbf1c0ec5aeb118a47a5f641525434ec7 diff --git a/image-tiff b/image-tiff new file mode 160000 index 0000000..96d37bb --- /dev/null +++ b/image-tiff @@ -0,0 +1 @@ +Subproject commit 96d37bb087f7eb1d1a3c495b13ad889914061da3 diff --git a/ndbioimage/Cargo.toml b/ndbioimage/Cargo.toml new file mode 100644 index 0000000..57cf7f9 --- /dev/null +++ b/ndbioimage/Cargo.toml @@ -0,0 +1,82 @@ +[package] +name = "ndbioimage" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +description.workspace = true +homepage.workspace = true +repository.workspace = true +documentation.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +exclude = ["/tests"] + +[lib] +name = "ndbioimage" +crate-type = ["cdylib", "rlib"] + +[dependencies] +bioformats = { version = "0.1", optional = true, default-features = false, features = ["zarr", "tissuefaxs"] } +clap = { version = "4", features = ["derive"] } +console = { version = "0.16", optional = true } +downloader = { version = "0.2", optional = true, default-features = false, features = ["rustls-tls"] } +ffmpeg-sidecar = { version = "2", optional = true } +itertools = "0.15" +image-registration = { path = "../image-registration", version = "0.1", optional = true } +indexmap = { version = "2", features = ["serde"] } +indicatif = { version = "0.18", features = ["rayon"], optional = true } +j4rs = { version = "0.25", optional = true } +libczirw-sys = { version = "0.5", optional = true } +ndarray = { version = "0.17", features = ["serde"] } +num = "0.4" +numpy = { version = "0.29", optional = true } +ome-metadata = "0.5" +ordered-float = { version = "5", optional = true } +phf = { version = "0.14", features = ["macros"] } +postcard = { version = "1", features = ["use-std"], optional = true } +pyo3 = { version = "0.29", features = ["abi3-py310", "anyhow", "generate-import-lib"], optional = true } +pyo3-stub-gen = { version = "0.23", optional = true } +rayon = { version = "1", optional = true } +regex = "1" +serde = { version = "1", features = ["rc", "derive"] } +serde_with = "3" +strum = { version = "0.28", features = ["derive"] } +thiserror = "2" +tiff = { package = "ndbioimage-tiff", path = "../image-tiff", version = "0.11", features = ["zstd"], optional = true } +tiffwrite = { version = "2026.6.0", optional = true } +tokio = { version = "1", features = ["rt", "rt-multi-thread"], optional = true } +thread_local = { version = "1", optional = true } +xmltree = { version = "0.12", optional = true } +yaml_serde = { version = "0.10", optional = true } + +[dev-dependencies] +rayon = "1" +regex = "1" +postcard = { version = "1", features = ["use-std"] } + +[build-dependencies] +j4rs = "0.25" +ffmpeg-sidecar = "2" +retry = "2" +toml = "1" + +[features] +default = ["bioformats_java", "gpl-formats", "czi", "tiff", "tiffseq", "movie", "tiffwrite"] +all = ["bioformats_java", "bioformats_rust", "czi", "gpl-formats", "movie", "tiffseq", "tiffwrite", "tiff", "transforms"] +gpl-formats = [] +python = ["dep:pyo3", "dep:numpy", "dep:pyo3-stub-gen", "dep:postcard", "ome-metadata/python"] +czi = ["dep:libczirw-sys", "dep:xmltree", "dep:thread_local"] +bioformats_rust = ["dep:bioformats", "dep:thread_local"] +bioformats_java = ["dep:j4rs", "dep:thread_local", "dep:downloader"] +tiffwrite = ["dep:tiffwrite", "dep:indicatif", "dep:console", "dep:rayon"] +tiffseq = ["dep:tiff", "dep:yaml_serde"] +tiff = ["dep:tiff", "dep:thread_local"] +movie = ["dep:ffmpeg-sidecar", "dep:tokio", "dep:ordered-float", "dep:indicatif", "dep:console"] +transforms = ["dep:image-registration", "dep:yaml_serde"] + +[package.metadata.docs.rs] +no-default-features = true +features = ["bioformats_java", "bioformats_rust", "czi", "movie", "tiffseq", "tiffwrite", "tiff", "transforms"] diff --git a/README.md b/ndbioimage/README.md similarity index 100% rename from README.md rename to ndbioimage/README.md diff --git a/build.rs b/ndbioimage/build.rs similarity index 84% rename from build.rs rename to ndbioimage/build.rs index d5fb558..c7c413f 100644 --- a/build.rs +++ b/ndbioimage/build.rs @@ -9,21 +9,41 @@ mod bioformats { #[derive(Clone, Debug)] pub(crate) enum BuildError { J4rsVersionNotFound, + LockFileNotFound, } impl Display for BuildError { fn fmt(&self, fmt: &mut Formatter) -> Result<(), std::fmt::Error> { match self { Self::J4rsVersionNotFound => write!(fmt, "J4rsVersion not found in Cargo.lock"), + Self::LockFileNotFound => write!( + fmt, + "Cargo.lock not found in any parent of CARGO_MANIFEST_DIR ({})", + std::env::var("CARGO_MANIFEST_DIR").unwrap_or_default() + ), } } } impl Error for BuildError {} + /// the `Cargo.lock` that cargo resolved for this build: next to the manifest for a standalone + /// package, or at the workspace root when this crate is a workspace member + fn lock_file() -> Result { + let manifest_dir = std::path::PathBuf::from( + std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set"), + ); + manifest_dir + .ancestors() + .map(|dir| dir.join("Cargo.lock")) + .find(|lock| lock.is_file()) + .ok_or(BuildError::LockFileNotFound) + } + fn get_j4rs_version() -> Result> { - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set"); - let lock_path = std::path::Path::new(&manifest_dir).join("Cargo.lock"); + let lock_path = lock_file()?; + // the resolved j4rs version is baked into the generated constants, so rerun when it changes + println!("cargo::rerun-if-changed={}", lock_path.display()); let lock_toml = std::fs::read_to_string(&lock_path)?; let value: toml::Value = toml::from_str(&lock_toml)?; if let Some(packages) = value.get("package").and_then(|v| v.as_array()) { diff --git a/py/ndbioimage/__init__.py b/ndbioimage/py/ndbioimage/__init__.py similarity index 83% rename from py/ndbioimage/__init__.py rename to ndbioimage/py/ndbioimage/__init__.py index a52af34..0522ee3 100755 --- a/py/ndbioimage/__init__.py +++ b/ndbioimage/py/ndbioimage/__init__.py @@ -5,7 +5,7 @@ import sys import warnings from importlib.metadata import version from pathlib import Path -from typing import TypeVar +from typing import TYPE_CHECKING import numpy as np from numpy.typing import ArrayLike @@ -13,14 +13,18 @@ from numpy.typing import ArrayLike os.environ["RUST_BACKTRACE"] = "full" os.environ["COLORBT_SHOW_HIDDEN"] = "1" -from .ndbioimage_rs import Imread, generate_stub +from .ndbioimage_rs import Imread, Shape, generate_stub -try: - from .ndbioimage_rs import transforms +if TYPE_CHECKING: + from .transforms import Transform as Transform +else: + try: + from .ndbioimage_rs import transforms - sys.modules["ndbioimage.transforms"] = transforms -except (ImportError, ModuleNotFoundError): - pass + sys.modules["ndbioimage.transforms"] = transforms + Transform = transforms.Transform + except (ImportError, ModuleNotFoundError): + pass try: from .ndbioimage_rs import batch_to_tiff @@ -50,7 +54,16 @@ except Exception: # noqa __git_commit_hash__ = "unknown" warnings.filterwarnings("ignore", "Reference to unknown ID") -Number = int | float | np.integer | np.floating + +__all__ = [ + "Imread", + "JVMException", + "ReaderNotFoundError", + "Shape", + "Transform", + "get_positions", + "is_noise", +] # backwards compatibility @@ -90,9 +103,6 @@ def ndbioimage_generate_stub(): (nd / "ndbioimage.pyi").rename(nd / "__init__.pyi") -R = TypeVar("R") - - def get_positions(path: str | Path) -> set[int]: return Imread.get_available_series(path) diff --git a/py/ndbioimage/ndbioimage_rs/__init__.pyi b/ndbioimage/py/ndbioimage/ndbioimage_rs/__init__.pyi similarity index 96% rename from py/ndbioimage/ndbioimage_rs/__init__.pyi rename to ndbioimage/py/ndbioimage/ndbioimage_rs/__init__.pyi index 9b2e158..0f3ba00 100644 --- a/py/ndbioimage/ndbioimage_rs/__init__.pyi +++ b/ndbioimage/py/ndbioimage/ndbioimage_rs/__init__.pyi @@ -228,7 +228,26 @@ class Imread: reader: typing.Optional[builtins.str] = None, ) -> Imread: r""" - new view on a file at path, open series #, open as dtype: (u)int(8/16/32) or float(32/64) + new view on an image file + path: path to image file to open, can include /Pos\d + dtype: when not None; will cast image to this dtype, + choices: (u)int(8/16/32) or float(32/64) + axes: the resulting object will represent the image with the axes in this order + reader: use a specific reader to open the file, + view available choices with Imread.available_readers + """ + @staticmethod + def warm_jvm() -> None: + r""" + creates the jvm (if it does not exist yet) so that hotspot installs its signal handlers before + anything else does: see `crate::readers::bioformats_java::warm_jvm` + + in a python process, call this before `ray.init()`, which installs CPython's faulthandler + """ + @staticmethod + def jvm_is_running() -> builtins.bool: + r""" + returns whether the jvm was already created on this thread """ def set_flags( self, @@ -773,7 +792,7 @@ def batch_to_tiff( def generate_stub(dest_path: builtins.str) -> None: r""" - generates ndbioimage/__init__.pyi + generates .pyi files """ def main() -> None: diff --git a/py/ndbioimage/ndbioimage_rs/transforms.pyi b/ndbioimage/py/ndbioimage/ndbioimage_rs/transforms.pyi similarity index 94% rename from py/ndbioimage/ndbioimage_rs/transforms.pyi rename to ndbioimage/py/ndbioimage/ndbioimage_rs/transforms.pyi index f1d85e1..284c30a 100644 --- a/py/ndbioimage/ndbioimage_rs/transforms.pyi +++ b/ndbioimage/py/ndbioimage/ndbioimage_rs/transforms.pyi @@ -120,6 +120,16 @@ class Transform: @center.setter def center(self, value: typing.Sequence[builtins.float]) -> None: ... @property + def origin(self) -> builtins.list[builtins.float]: + r""" + backwards compatibility + """ + @origin.setter + def origin(self, value: typing.Sequence[builtins.float]) -> None: + r""" + backwards compatibility + """ + @property def shape(self) -> builtins.list[builtins.int]: ... @shape.setter def shape(self, value: typing.Sequence[builtins.int]) -> None: ... @@ -184,6 +194,12 @@ class Transform: def interpolate_par( self, order: builtins.int, image: numpy.typing.ArrayLike ) -> numpy.typing.NDArray[numpy.float64]: ... + def frame( + self, image: numpy.typing.ArrayLike, default: typing.Optional[typing.Any] = None + ) -> numpy.ndarray: + r""" + backwards compatibility, default parameter is not implemented + """ def is_unity(self) -> builtins.bool: ... def transform_point( self, point: numpy.typing.ArrayLike diff --git a/py/ndbioimage/py.typed b/ndbioimage/py/ndbioimage/py.typed similarity index 100% rename from py/ndbioimage/py.typed rename to ndbioimage/py/ndbioimage/py.typed diff --git a/py/ndbioimage/transforms.pyi b/ndbioimage/py/ndbioimage/transforms.pyi similarity index 100% rename from py/ndbioimage/transforms.pyi rename to ndbioimage/py/ndbioimage/transforms.pyi diff --git a/pyproject.toml b/ndbioimage/pyproject.toml similarity index 100% rename from pyproject.toml rename to ndbioimage/pyproject.toml diff --git a/src/axes.rs b/ndbioimage/src/axes.rs similarity index 100% rename from src/axes.rs rename to ndbioimage/src/axes.rs diff --git a/src/cache.rs b/ndbioimage/src/cache.rs similarity index 72% rename from src/cache.rs rename to ndbioimage/src/cache.rs index c8c581a..f74f2e0 100644 --- a/src/cache.rs +++ b/ndbioimage/src/cache.rs @@ -6,6 +6,7 @@ use std::fmt::{Debug, Formatter}; use std::hash::{Hash, Hasher}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, OnceLock}; +use std::time::SystemTime; use crate::readers::Frame; @@ -15,6 +16,68 @@ const DEFAULT_FRAME_CACHE_SIZE: usize = 128; /// maximum number of materialized arrays held in the cache const DEFAULT_ARRAY_CACHE_SIZE: usize = 32; +/// fingerprint of the file(s) a reader reads from. +/// +/// The cache is keyed partly on this stamp, so a file that is rewritten (for +/// example an output file that is written and then read back) gets a new cache +/// key instead of returning stale pixels. The stamp is read from the +/// filesystem on every lookup, so a long-lived view also notices a change. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +pub(crate) struct FileStamp { + /// size of the file in bytes + len: u64, + /// last modification time + modified: Option, + /// last status change time; on unix this catches same-size rewrites that + /// happen within the timestamp resolution of `modified` + #[cfg(unix)] + changed: Option<(i64, u32)>, + /// inode number; on unix this catches an atomic replace that preserves the + /// modification time and size + #[cfg(unix)] + inode: u64, + /// creation time; windows has no stable `ctime` or file index, but this + /// changes when a file is deleted/recreated or atomically replaced, which is + /// the case the unix inode covers. in-place rewrites are caught by + /// `modified`, which has 100 ns resolution on windows. + #[cfg(windows)] + created: Option, +} + +impl FileStamp { + /// read the current stamp for `path`, returning the default (all-zero) + /// stamp when the metadata cannot be read + pub(crate) fn from_path(path: &Path) -> Self { + let Ok(meta) = std::fs::metadata(path) else { + return Self::default(); + }; + let len = meta.len(); + let modified = meta.modified().ok(); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + Self { + len, + modified, + changed: Some((meta.ctime(), meta.ctime_nsec() as u32)), + inode: meta.ino(), + } + } + #[cfg(windows)] + { + Self { + len, + modified, + created: meta.created().ok(), + } + } + #[cfg(not(any(unix, windows)))] + { + Self { len, modified } + } + } +} + /// identity of the reader a frame was read from #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub(crate) struct ReaderKey { @@ -22,6 +85,7 @@ pub(crate) struct ReaderKey { pub(crate) path: PathBuf, pub(crate) series: usize, pub(crate) position: usize, + pub(crate) stamp: FileStamp, } /// borrowed view of [`ReaderKey`] for cache lookups without allocation. @@ -33,6 +97,7 @@ pub(crate) struct ReaderKeyRef<'a> { pub(crate) path: &'a Path, pub(crate) series: usize, pub(crate) position: usize, + pub(crate) stamp: FileStamp, } impl Hash for ReaderKeyRef<'_> { @@ -41,6 +106,7 @@ impl Hash for ReaderKeyRef<'_> { self.path.hash(state); self.series.hash(state); self.position.hash(state); + self.stamp.hash(state); } } @@ -50,6 +116,7 @@ impl Equivalent for ReaderKeyRef<'_> { && self.path == key.path.as_path() && self.series == key.series && self.position == key.position + && self.stamp == key.stamp } } @@ -60,6 +127,7 @@ impl ReaderKeyRef<'_> { path: self.path.to_path_buf(), series: self.series, position: self.position, + stamp: self.stamp, } } } @@ -320,3 +388,42 @@ impl Debug for ArrayCache { f.debug_struct("ArrayCache").finish_non_exhaustive() } } + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_path(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("ndbioimage-cache-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir.join(name) + } + + #[test] + fn file_stamp_detects_rewrite() { + let path = temp_path("stamp.bin"); + std::fs::write(&path, b"aaaa").unwrap(); + let before = FileStamp::from_path(&path); + std::fs::write(&path, b"bbbbbb").unwrap(); + let after = FileStamp::from_path(&path); + assert_ne!(before, after); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn reader_key_includes_file_stamp() { + let key = |stamp| ReaderKey { + name: "test".to_string(), + path: PathBuf::from("test.tif"), + series: 0, + position: 0, + stamp, + }; + let a = key(FileStamp::from_path(Path::new("does-not-exist.tif"))); + let b = key(FileStamp { + len: 1, + ..FileStamp::default() + }); + assert_ne!(a, b); + } +} diff --git a/src/colors.rs b/ndbioimage/src/colors.rs similarity index 100% rename from src/colors.rs rename to ndbioimage/src/colors.rs diff --git a/src/error.rs b/ndbioimage/src/error.rs similarity index 94% rename from src/error.rs rename to ndbioimage/src/error.rs index a75a237..ea74b58 100644 --- a/src/error.rs +++ b/ndbioimage/src/error.rs @@ -47,7 +47,7 @@ pub enum Error { #[error(transparent)] Tiff(#[from] tiff::TiffError), /// a postcard (de)serialization error - #[cfg(feature = "python")] + #[cfg(any(test, feature = "python"))] #[error(transparent)] PostCard(#[from] postcard::Error), /// an error from the libczi binding @@ -172,6 +172,13 @@ pub enum Error { /// file mismatch #[error("{0} do not match in {1} and {2}")] FileMismatch(String, String, String), + /// a jvm created in another process was inherited across a fork + #[cfg(feature = "bioformats_java")] + #[error( + "the jvm was created in process {0} but this is process {1}: a forked child inherits a jvm \ + it cannot use. read the file before forking, or start the child with spawn/exec instead." + )] + ForkedJvm(u32, u32), } impl Error { diff --git a/src/lib.rs b/ndbioimage/src/lib.rs similarity index 100% rename from src/lib.rs rename to ndbioimage/src/lib.rs diff --git a/src/main.rs b/ndbioimage/src/main.rs similarity index 100% rename from src/main.rs rename to ndbioimage/src/main.rs diff --git a/src/metadata.rs b/ndbioimage/src/metadata.rs similarity index 100% rename from src/metadata.rs rename to ndbioimage/src/metadata.rs diff --git a/src/movie.rs b/ndbioimage/src/movie.rs similarity index 100% rename from src/movie.rs rename to ndbioimage/src/movie.rs diff --git a/src/py.rs b/ndbioimage/src/py.rs similarity index 98% rename from src/py.rs rename to ndbioimage/src/py.rs index 4ba9737..19486b6 100644 --- a/src/py.rs +++ b/ndbioimage/src/py.rs @@ -40,7 +40,7 @@ pub(crate) fn ndbioimage_file() -> PathBuf { PathBuf::from(file) } -/// generates ndbioimage/__init__.pyi +/// generates .pyi files #[gen_stub_pyfunction] #[pyfunction] pub fn generate_stub(dest_path: String) -> PyResult<()> { diff --git a/src/py/imread.rs b/ndbioimage/src/py/imread.rs similarity index 98% rename from src/py/imread.rs rename to ndbioimage/src/py/imread.rs index 9b4134d..55fb7d0 100644 --- a/src/py/imread.rs +++ b/ndbioimage/src/py/imread.rs @@ -147,7 +147,13 @@ impl PyView { #[gen_stub_pymethods] #[pymethods] impl PyView { - /// new view on a file at path, open series #, open as dtype: (u)int(8/16/32) or float(32/64) + /// new view on an image file + /// path: path to image file to open, can include /Pos\d + /// dtype: when not None; will cast image to this dtype, + /// choices: (u)int(8/16/32) or float(32/64) + /// axes: the resulting object will represent the image with the axes in this order + /// reader: use a specific reader to open the file, + /// view available choices with Imread.available_readers #[new] #[pyo3(signature = (path, dtype = None, axes = "cztyx", reader = None))] fn new<'py>( @@ -206,6 +212,26 @@ impl PyView { } } + /// creates the jvm (if it does not exist yet) so that hotspot installs its signal handlers before + /// anything else does: see `crate::readers::bioformats_java::warm_jvm` + /// + /// in a python process, call this before `ray.init()`, which installs CPython's faulthandler + #[cfg(feature = "bioformats_java")] + #[staticmethod] + fn warm_jvm() -> PyResult<()> { + Ok(crate::readers::bioformats_java::warm_jvm()?) + } + + /// returns whether the jvm was already created in this process + /// + /// this is false in a process that inherited a jvm across a fork, since such a jvm cannot be + /// used; reading a bio-formats file there raises instead + #[cfg(feature = "bioformats_java")] + #[staticmethod] + fn jvm_is_running() -> bool { + crate::readers::bioformats_java::jvm_is_running() + } + /// https://numpy.org/doc/stable/reference/generated/numpy.ndarray.flags.html #[getter] fn flags(&self) -> PyFlags { diff --git a/src/py/transforms.rs b/ndbioimage/src/py/transforms.rs similarity index 89% rename from src/py/transforms.rs rename to ndbioimage/src/py/transforms.rs index c6abab8..f488ab3 100644 --- a/src/py/transforms.rs +++ b/ndbioimage/src/py/transforms.rs @@ -5,9 +5,11 @@ use image_registration::register::{Optimizer, RegistrationStep}; use image_registration::transform::Transform; use ndarray::{Ix0, Ix1, Ix2, Ix3, Ix4, Ix5, IxDyn}; use numpy::{ - AllowTypeChange, IntoPyArray, PyArray, PyArray1, PyArray2, PyArrayDyn, PyArrayLike1, - PyArrayLike2, PyArrayLikeDyn, + AllowTypeChange, IntoPyArray, PyArray, PyArray1, PyArray2, PyArrayDescrMethods, PyArrayDyn, + PyArrayLike1, PyArrayLike2, PyArrayLikeDyn, PyArrayMethods, PyUntypedArray, + PyUntypedArrayMethods, dtype, }; +use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PyNotImplementedError, PyValueError}; use pyo3::prelude::*; use pyo3_stub_gen::derive::*; @@ -165,6 +167,18 @@ impl PyTransform { self.inner.center = center; } + /// backwards compatibility + #[getter] + fn get_origin(&self) -> Vec { + self.inner.center.clone() + } + + /// backwards compatibility + #[setter] + fn set_origin(&mut self, center: Vec) { + self.inner.center = center; + } + #[getter] fn get_shape(&self) -> Vec { self.inner.shape.clone() @@ -329,6 +343,88 @@ impl PyTransform { .into_pyarray(py)) } + /// backwards compatibility, default parameter is not implemented + #[gen_stub(override_return_type(type_repr="numpy.ndarray", imports=("numpy")))] + #[pyo3(signature = (image, default = None))] + fn frame<'py>( + &self, + py: Python<'py>, + #[gen_stub(override_type(type_repr="numpy.typing.ArrayLike", imports=("numpy.typing")))] + image: Bound<'py, PyUntypedArray>, + default: Option>, + ) -> PyResult> { + if default.is_some() { + return Err(PyNotImplementedError::new_err( + "default parameter is not implemented", + )); + } + if image.ndim() != self.inner.ndim() { + return Err(PyValueError::new_err(format!( + "dimensionality of image ({}) and transform ({}) do not correspond", + image.ndim(), + self.inner.ndim() + ))); + } + + macro_rules! interp { + ($order:expr, $t:tt) => { + self.inner + .interpolate_par::<$order, _, _>( + unsafe { image.cast::>()?.as_array() } + .mapv(|i| i as f64) + .into_dimensionality() + .map_err(Error::from)? + .view(), + ) + .map_err(Error::from)? + .mapv(|i| i as $t) + .into_pyarray(py) + .into_bound_py_any(py) + }; + } + + let dt = image.dtype(); + if dt.is_equiv_to(&dtype::(py)) { + self.inner + .interpolate_par::<0, _, _>( + unsafe { image.cast::>()?.as_array() } + .mapv(|i| if i { 1.0 } else { 0.0 }) + .into_dimensionality() + .map_err(Error::from)? + .view(), + ) + .map_err(Error::from)? + .mapv(|i| i > 0.5) + .into_pyarray(py) + .into_bound_py_any(py) + } else if dt.is_equiv_to(&dtype::(py)) { + interp!(0, u8) + } else if dt.is_equiv_to(&dtype::(py)) { + interp!(0, i8) + } else if dt.is_equiv_to(&dtype::(py)) { + interp!(0, u16) + } else if dt.is_equiv_to(&dtype::(py)) { + interp!(0, i16) + } else if dt.is_equiv_to(&dtype::(py)) { + interp!(0, u32) + } else if dt.is_equiv_to(&dtype::(py)) { + interp!(0, i32) + } else if dt.is_equiv_to(&dtype::(py)) { + interp!(0, u64) + } else if dt.is_equiv_to(&dtype::(py)) { + interp!(0, i64) + } else if dt.is_equiv_to(&dtype::(py)) { + interp!(3, f32) + } else if dt.is_equiv_to(&dtype::(py)) { + interp!(3, f64) + } else { + Err(PyValueError::new_err(format!( + "invalid dtype: {}; must be bool, (u)int(8/16/32/64) or float(32/64)", + dt + ))) + } + } + fn is_unity(&self) -> bool { self.inner.is_unity() } diff --git a/src/readers.rs b/ndbioimage/src/readers.rs similarity index 100% rename from src/readers.rs rename to ndbioimage/src/readers.rs diff --git a/src/readers/bioformats_java.rs b/ndbioimage/src/readers/bioformats_java.rs similarity index 74% rename from src/readers/bioformats_java.rs rename to ndbioimage/src/readers/bioformats_java.rs index 306ee38..b8b47cb 100644 --- a/src/readers/bioformats_java.rs +++ b/ndbioimage/src/readers/bioformats_java.rs @@ -14,6 +14,7 @@ use std::hash::{Hash, Hasher}; use std::ops::Deref; use std::rc::Rc; use std::sync::Mutex; +use std::sync::atomic::{AtomicU32, Ordering}; use thread_local::ThreadLocal; include!(concat!(env!("OUT_DIR"), "/constants.rs")); @@ -24,10 +25,33 @@ thread_local! { static DOWNLOAD_LOCK: Mutex<()> = Mutex::new(()); static JVM_BUILT: Mutex = Mutex::new(false); +/// the process that called [JvmBuilder::build], 0 if no jvm has been built yet +static JVM_PID: AtomicU32 = AtomicU32::new(0); + +/// Returns `Ok(())` if the jvm in this process can still be used, and an error if it was inherited +/// across a fork. +/// +/// A fork copies the address space, so the child gets the parent's jvm wholesale: its threads do +/// not exist and any lock another thread held at the moment of the fork stays held forever. There +/// is no way to repair that, and no way to start a replacement either, because HotSpot's +/// "a jvm already exists" flag lives in libjvm's private data and is copied along with everything +/// else -- verified by `JNI_GetCreatedJavaVMs`, which still reports 1 in the child, so a second +/// `JNI_CreateJavaVM` (which is what j4rs' `Jvm::attach_thread` calls) takes the already-exists +/// path. An `exec` replaces the address space, and that does reset the count to 0. +fn check_not_forked() -> Result<(), Error> { + let built_in = JVM_PID.load(Ordering::Relaxed); + if built_in != 0 && built_in != std::process::id() { + return Err(Error::ForkedJvm(built_in, std::process::id())); + } + Ok(()) +} /// Ensure 1 jvm per thread -fn jvm() -> Rc { - JVM.with(|cell| { +fn jvm() -> Result, Error> { + // before touching JVM: in a forked child that thread's cell may already hold an Rc whose + // vm is gone, and the guard in check_not_forked is the only thing that can catch that. + check_not_forked()?; + Ok(JVM.with(|cell| { cell.get_or_init(move || { #[cfg(feature = "python")] let path = crate::py::ndbioimage_file(); @@ -113,6 +137,7 @@ fn jvm() -> Rc { .java_opt(JavaOpt::new("--sun-misc-unsafe-memory-access=allow")) .build() .expect("Failed to build JVM"); + JVM_PID.store(std::process::id(), Ordering::Relaxed); if let Ok(e) = InvocationArg::try_from("ERROR") { let _ = j.invoke_static( "loci.common.DebugTools", @@ -125,9 +150,103 @@ fn jvm() -> Rc { } }) .clone() - }) + })) } +/// Returns `true` if the jvm was already created on the current thread. +/// +/// The jvm is created lazily, on the first bio-formats read, and HotSpot installs its own signal +/// handlers while starting up. So this tells whether those handlers are in place yet. +/// +/// This is process-wide, not thread-local: a forked child inherited the parent's jvm but cannot +/// use it, so reporting `true` there would only invite the deadlock that [check_not_forked] turns +/// into an error. +pub fn jvm_is_running() -> bool { + JVM_PID.load(Ordering::Relaxed) == std::process::id() && JVM.with(|cell| cell.get().is_some()) +} + +/// Creates the jvm on the current thread if it does not exist yet (no-op if it already does), +/// and initializes bio-formats on it. +/// +/// This exists because of how HotSpot probes the cpu features during VM init: on an avx-512 +/// capable cpu it deliberately reads through a NULL pointer (`VM_Version::_cpuinfo_segv_addr`, set +/// to the address of the generated cpu-feature stub) and resumes from its own SIGSEGV handler after +/// recognizing that address in `PosixSignals::pd_hotspot_signal_handler`. +/// +/// Any SIGSEGV handler that was installed *before* the VM started is not that handler, and +/// HotSpot chains to it instead. Then the deliberate fault is not recognized, the chained handler +/// re-raises SIGSEGV, and the re-entrant pass no longer has a recognizable pc and ends in +/// `VMError::report_and_die` -> `os::die()` -> `abort()`. +/// +/// In a python process this bites when `ray.init()` has already installed CPython's faulthandler +/// (ray/_private/worker.py), so call this before that. +/// +/// Note that creating the VM is *not* enough: it has to be initialized far enough that the +/// deliberate fault has been taken before the foreign handler shows up. Measured on an AMD EPYC +/// 9124 (avx-512, XCR0 = 0x2e7) with ray: merely creating the VM leaves the crash rate unchanged, +/// and so does loading the reader classes without opening a file. What does work is initializing a +/// reader on an actual (tiny) image, so that is what this does. +pub fn warm_jvm() -> Result<(), Error> { + let jvm = jvm()?; + let no_args: &[InvocationArg] = &[]; + // jvm() only *creates* the VM; run one static method so that this only returns Ok once the VM + // can actually run java bytecode. + let version = jvm.invoke_static( + "java.lang.System", + "getProperty", + &[InvocationArg::try_from("java.version")?], + )?; + let _: String = jvm.to_rust(version)?; + + // loads (and runs the static initializers of) every reader class in the registry + let _ = jvm.invoke_static( + "loci.formats.ImageReader", + "getDefaultReaderClasses", + no_args, + )?; + + // then open a 1x1 image so that the reader and its OMEXML metadata store get initialized. + // ImageReader.setId needs a real file, so write the tiff out and clean it up again. + let path = std::env::temp_dir().join(format!("ndbioimage-warm-jvm-{}.tif", std::process::id())); + std::fs::write(&path, WARM_TIFF)?; + let result = (|| -> Result<(), Error> { + let reader = jvm.create_instance("loci.formats.ImageReader", no_args)?; + jvm.invoke( + &reader, + "setId", + &[InvocationArg::try_from(path.to_string_lossy().as_ref())?], + )?; + let _: i32 = jvm.to_rust(jvm.invoke(&reader, "getSizeY", no_args)?)?; + let _ = jvm.invoke(&reader, "getMetadataStore", no_args)?; + Ok(()) + })(); + let _ = std::fs::remove_file(&path); + result +} + +/// A 1x1 8-bit grayscale tiff (written by tifffile), used by [warm_jvm] to give bio-formats +/// something it can open. +const WARM_TIFF: &[u8] = &[ + 0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x01, 0x04, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x01, 0x04, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x02, 0x01, 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x03, 0x01, + 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, 0x01, 0x03, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0e, 0x01, 0x02, 0x00, 0x12, 0x00, 0x00, 0x00, 0xb6, 0x00, + 0x00, 0x00, 0x11, 0x01, 0x04, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x15, 0x01, + 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x16, 0x01, 0x04, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x17, 0x01, 0x04, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x1a, 0x01, 0x05, 0x00, 0x01, 0x00, 0x00, 0x00, 0xd8, 0x00, 0x00, 0x00, 0x1b, 0x01, + 0x05, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0x28, 0x01, 0x03, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x31, 0x01, 0x02, 0x00, 0x0c, 0x00, 0x00, 0x00, 0xe8, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x22, 0x73, 0x68, 0x61, 0x70, 0x65, 0x22, 0x3a, + 0x20, 0x5b, 0x31, 0x2c, 0x20, 0x31, 0x5d, 0x7d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x74, 0x69, 0x66, 0x66, 0x66, 0x69, 0x6c, 0x65, 0x2e, 0x70, 0x79, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, +]; + macro_rules! method_return { ($R:ty$(|c)?) => { Result<$R, Error> }; () => { Result<(), Error> }; @@ -147,14 +266,14 @@ macro_rules! method { #[allow(dead_code)] pub(crate) fn $name(&self, $($($n: $t),*)?) -> method_return!($($tt)?) { let args: Vec = vec![$($( method_arg!($n:$t$(|$p)?) ),*)?]; - let _result = jvm().invoke(&self.0, $method, &args)?; + let _result = jvm()?.invoke(&self.0, $method, &args)?; macro_rules! method_result { ($R:ty|c) => { - Ok(jvm().to_rust(_result)?) + Ok(jvm()?.to_rust(_result)?) }; ($R:ty|d) => { - Ok(jvm().to_rust_deserialized(_result)?) + Ok(jvm()?.to_rust_deserialized(_result)?) }; ($R:ty) => { Ok(_result) @@ -187,7 +306,7 @@ pub struct DebugTools; impl DebugTools { /// set debug root level: ERROR, DEBUG, TRACE, INFO, OFF pub fn set_root_level(level: &str) -> Result<(), Error> { - jvm().invoke_static( + jvm()?.invoke_static( "loci.common.DebugTools", "setRootLevel", &[InvocationArg::try_from(level)?], @@ -201,7 +320,7 @@ pub(crate) struct ChannelSeparator(Instance); impl ChannelSeparator { pub(crate) fn new(image_reader: &ImageReader) -> Result { - let jvm = jvm(); + let jvm = jvm()?; let channel_separator = jvm.create_instance( "loci.formats.ChannelSeparator", &[InvocationArg::from(jvm.clone_instance(&image_reader.0)?)], @@ -228,7 +347,7 @@ impl Drop for ImageReader { impl ImageReader { pub(crate) fn new() -> Result { - let reader = jvm().create_instance("loci.formats.ImageReader", InvocationArg::empty())?; + let reader = jvm()?.create_instance("loci.formats.ImageReader", InvocationArg::empty())?; Ok(ImageReader(reader)) } @@ -238,7 +357,7 @@ impl ImageReader { pub(crate) fn ome_xml(&self) -> Result { let mds = self.get_metadata_store()?; - Ok(jvm() + Ok(jvm()? .chain(&mds)? .cast("loci.formats.ome.OMEPyramidStore")? .invoke("dumpXML", InvocationArg::empty())? @@ -276,7 +395,7 @@ pub(crate) struct MetadataTools(Instance); impl MetadataTools { pub(crate) fn new() -> Result { let meta_data_tools = - jvm().create_instance("loci.formats.MetadataTools", InvocationArg::empty())?; + jvm()?.create_instance("loci.formats.MetadataTools", InvocationArg::empty())?; Ok(MetadataTools(meta_data_tools)) } @@ -517,6 +636,10 @@ impl Reader for BioFormatsJavaReader { where P: AsRef, { + // checked here as well as in jvm() so that this entry point fails with the error rather + // than reaching get_size_x() below, which goes through deref() -> get_reader().unwrap() + // and would turn it into a panic. + check_not_forked()?; let mut path = path.as_ref().to_path_buf(); if path.is_dir() { let orig = path.clone(); @@ -648,6 +771,7 @@ mod tests { metadata_g: "tiffseq/4-Pos_001_002/img_000000000_Cy3-Cy3_filter_000.tif", metadata_h: "tiffseq/20-Pos_005_005/img_000000000_Cy3-Cy3_filter_000.tif", metadata_i: "tiffseq/YTL1841B2-2-1_1hr_DMSO_galinduction_1", + metadata_j: "tiff/R1_NRF1_Gtf3a500nM_REAL_20260731_25pct_500ms_001_MMStack_PosR1_NRF1_Gtf3a500nM_REAL_20260731_25pct_500ms_001.ome.tif", } #[test] diff --git a/src/readers/bioformats_rust.rs b/ndbioimage/src/readers/bioformats_rust.rs similarity index 100% rename from src/readers/bioformats_rust.rs rename to ndbioimage/src/readers/bioformats_rust.rs diff --git a/src/readers/czi.rs b/ndbioimage/src/readers/czi.rs similarity index 100% rename from src/readers/czi.rs rename to ndbioimage/src/readers/czi.rs diff --git a/src/readers/tiff.rs b/ndbioimage/src/readers/tiff.rs similarity index 86% rename from src/readers/tiff.rs rename to ndbioimage/src/readers/tiff.rs index 260df0d..6101c6d 100644 --- a/src/readers/tiff.rs +++ b/ndbioimage/src/readers/tiff.rs @@ -7,10 +7,11 @@ use std::cell::{RefCell, RefMut}; use std::collections::{HashMap, HashSet}; use std::fs::File; use std::hash::{Hash, Hasher}; +use std::io::{Read, Seek}; use std::path::{Path, PathBuf}; use thread_local::ThreadLocal; use tiff::decoder::ifd::Value; -use tiff::decoder::{Decoder, DecodingResult}; +use tiff::decoder::{Decoder, DecodingSampleBuffer}; use tiff::tags::Tag; /// reader for single tiff image files @@ -70,8 +71,9 @@ impl TiffReader { fn get_reader(&self) -> Result>, Error> { self.reader .get_or_try(|| { - let file = File::open(&self.path)?; - Ok(RefCell::new(Decoder::new(file)?)) + let mut decoder = Decoder::open(File::open(&self.path)?)?; + decoder.next_image()?; + Ok(RefCell::new(decoder)) }) .map(|i| i.borrow_mut()) } @@ -128,10 +130,11 @@ impl Reader for TiffReader { }; let mut reader = new.get_reader()?; - let width = reader.get_tag(Tag::ImageWidth)?.into_u32()? as usize; - let height = reader.get_tag(Tag::ImageLength)?.into_u32()? as usize; + let width = reader.current_ifd().get_tag(Tag::ImageWidth)?.into_u32()? as usize; + let height = reader.current_ifd().get_tag(Tag::ImageLength)?.into_u32()? as usize; let n_samples = reader + .current_ifd() .get_tag(Tag::SamplesPerPixel) .ok() .and_then(|t| t.into_u16().ok()) @@ -139,9 +142,14 @@ impl Reader for TiffReader { let bits = tag_first_u16(&mut reader, Tag::BitsPerSample).unwrap_or(8); let sfmt = tag_first_u16(&mut reader, Tag::SampleFormat).unwrap_or(1); + + // indexed color pages (PhotometricInterpretation::RGBPalette) are read as plain + // intensities: the pixel values are indices into the ColorMap palette, which is display + // information that ndbioimage does not use. let pixtype = pixel_type_from_bits_sample(bits, sfmt)?; let planar = reader + .current_ifd() .get_tag(Tag::PlanarConfiguration) .ok() .and_then(|t| t.into_u16().ok()) @@ -149,8 +157,9 @@ impl Reader for TiffReader { == 2; let imagej_bytes = reader + .current_ifd() .get_tag(Tag::Unknown(50839)) - .or_else(|_| reader.get_tag(Tag::ImageDescription))?; + .or_else(|_| reader.current_ifd().get_tag(Tag::ImageDescription))?; let imagej_bytes = match imagej_bytes { Value::Ascii(s) => s.into_bytes(), other => other.into_u8_vec()?, @@ -180,7 +189,7 @@ impl Reader for TiffReader { let mut n_ifds = 1; while reader.more_images() { - reader.next_image()?; + reader.next_directory()?; n_ifds += 1; } @@ -212,8 +221,9 @@ impl Reader for TiffReader { fn metadata(&self) -> Result { let mut reader = self.get_reader()?; let imagej_bytes = reader + .current_ifd() .get_tag(Tag::Unknown(50839)) - .or_else(|_| reader.get_tag(Tag::ImageDescription))?; + .or_else(|_| reader.current_ifd().get_tag(Tag::ImageDescription))?; let imagej_bytes = match imagej_bytes { Value::Ascii(s) => s.into_bytes(), other => other.into_u8_vec()?, @@ -225,11 +235,13 @@ impl Reader for TiffReader { .get("interval") .and_then(|v| v.parse::().ok()); let ru = reader + .current_ifd() .get_tag(Tag::ResolutionUnit) .ok() .and_then(|i| i.into_u16().ok()) .unwrap_or(2); let pixel_size = reader + .current_ifd() .get_tag(Tag::XResolution) .ok() .and_then(|i| i.into_u64_vec().ok()) @@ -353,8 +365,8 @@ impl Reader for TiffReader { let mut reader = self.get_reader()?; reader.seek_to_image(page_idx)?; - let data: DecodingResult = if self.planar { - let mut result = DecodingResult::U16(Vec::new()); + let data: DecodingSampleBuffer = if self.planar { + let mut result = DecodingSampleBuffer::U16(Vec::new()); reader.read_image_to_buffer(&mut result)?; result } else { @@ -364,7 +376,7 @@ impl Reader for TiffReader { macro_rules! to_frame { ($var:ident, $ty:ty) => {{ let v: Vec<$ty> = match data { - DecodingResult::$var(v) => v, + DecodingSampleBuffer::$var(v) => v, _ => { return Err(Error::Parse(format!( "type mismatch: expected {}", @@ -448,9 +460,8 @@ impl Reader for TiffReader { } } -fn tag_first_u16(reader: &mut RefMut>, tag: Tag) -> Option { - let val = reader.get_tag(tag).ok()?; - use tiff::decoder::ifd::Value; +fn tag_first_u16(reader: &mut RefMut>, tag: Tag) -> Option { + let val = reader.current_ifd().get_tag(tag).ok()?; match val { Value::Short(v) => Some(v), Value::List(list) => list.first().and_then(|v| match v { @@ -464,6 +475,7 @@ fn tag_first_u16(reader: &mut RefMut>, tag: Tag) -> Option { #[cfg(test)] mod tests { use super::*; + use ndarray::s; fn open(file: &str) -> Result { let path = std::env::current_dir()? @@ -480,6 +492,22 @@ mod tests { fn $name() -> Result<(), Error> { let ts = open($file)?; println!("{}", ts.view().squeeze()?.summary()?); + + let view = ts.view(); + let first_frame = view.slice_cztyx(s![0, 0, 0, .., ..])?; + let array = first_frame.as_array::()?; + println!("{:?}", array); + assert_eq!(ts.shape.y, array.shape()[0]); + assert_eq!(ts.shape.x, array.shape()[1]); + + if view.shape.t > 1 { + let second_frame = view.slice_cztyx(s![0, 0, 1, .., ..])?; + let array = second_frame.as_array::()?; + println!("{:?}", array); + assert_eq!(ts.shape.y, array.shape()[0]); + assert_eq!(ts.shape.x, array.shape()[1]); + } + Ok(()) } )* @@ -487,11 +515,12 @@ mod tests { } test_metadata! { - // metadata_a: "tiff/20251014_20-Pos_000_000_mask.tif", + metadata_a: "tiff/20251014_20-Pos_000_000_mask.tif", metadata_b: "tiff/20251014_20-Pos_000_000_max.tif", metadata_c: "tiff/20251014_20-Pos_000_000_loc_results_Cy3.tif", metadata_e: "tiff/1xp53-01-AP1.tiff", metadata_f: "tiff/test.tif", - // metadata_g: "tiff/YTL1849A111_2023_05_04__14_46_19_cellnr_1_track.tif", + metadata_g: "tiff/YTL1849A111_2023_05_04__14_46_19_cellnr_1_track.tif", + metadata_h: "tiff/Wapl-AID_C46_13_tagRFPt_MCP_well1_video1_1_max.tif" } } diff --git a/src/readers/tiffseq.rs b/ndbioimage/src/readers/tiffseq.rs similarity index 95% rename from src/readers/tiffseq.rs rename to ndbioimage/src/readers/tiffseq.rs index 49b9f82..a392ed3 100644 --- a/src/readers/tiffseq.rs +++ b/ndbioimage/src/readers/tiffseq.rs @@ -8,7 +8,7 @@ use std::collections::{HashMap, HashSet}; use std::hash::{Hash, Hasher}; use std::path::{Path, PathBuf}; use std::str::FromStr; -use tiff::decoder::{Decoder, DecodingResult}; +use tiff::decoder::{Decoder, DecodingSampleBuffer}; use tiff::tags::Tag; /// reader for sequences of tiff files (one file per z/time plane) @@ -84,9 +84,13 @@ impl TiffSeqReader { fn read_tiff_dimensions(path: &Path) -> Result<(usize, usize), Error> { let file = std::fs::File::open(path)?; - let mut decoder = Decoder::new(file)?; - let width = decoder.get_tag(Tag::ImageWidth)?.into_u32()? as usize; - let height = decoder.get_tag(Tag::ImageLength)?.into_u32()? as usize; + let mut decoder = Decoder::open(file)?; + decoder.next_image()?; + let width = decoder.current_ifd().get_tag(Tag::ImageWidth)?.into_u32()? as usize; + let height = decoder + .current_ifd() + .get_tag(Tag::ImageLength)? + .into_u32()? as usize; Ok((width, height)) } } @@ -514,38 +518,39 @@ impl Reader for TiffSeqReader { .ok_or_else(|| Error::OutOfBounds(c.max(z).max(t) as isize, 0))?; let rdr = std::fs::File::open(file)?; - let mut decoder = Decoder::new(rdr)?; + let mut decoder = Decoder::open(rdr)?; + decoder.next_image()?; match decoder.read_image()? { - DecodingResult::U8(d) => Ok(ArrayT::U8(Array2::from_shape_vec( + DecodingSampleBuffer::U8(d) => Ok(ArrayT::U8(Array2::from_shape_vec( (self.shape.y, self.shape.x), d, )?)), - DecodingResult::U16(d) => Ok(ArrayT::U16(Array2::from_shape_vec( + DecodingSampleBuffer::U16(d) => Ok(ArrayT::U16(Array2::from_shape_vec( (self.shape.y, self.shape.x), d, )?)), - DecodingResult::U32(d) => Ok(ArrayT::U32(Array2::from_shape_vec( + DecodingSampleBuffer::U32(d) => Ok(ArrayT::U32(Array2::from_shape_vec( (self.shape.y, self.shape.x), d, )?)), - DecodingResult::I8(d) => Ok(ArrayT::I8(Array2::from_shape_vec( + DecodingSampleBuffer::I8(d) => Ok(ArrayT::I8(Array2::from_shape_vec( (self.shape.y, self.shape.x), d, )?)), - DecodingResult::I16(d) => Ok(ArrayT::I16(Array2::from_shape_vec( + DecodingSampleBuffer::I16(d) => Ok(ArrayT::I16(Array2::from_shape_vec( (self.shape.y, self.shape.x), d, )?)), - DecodingResult::I32(d) => Ok(ArrayT::I32(Array2::from_shape_vec( + DecodingSampleBuffer::I32(d) => Ok(ArrayT::I32(Array2::from_shape_vec( (self.shape.y, self.shape.x), d, )?)), - DecodingResult::F32(d) => Ok(ArrayT::F32(Array2::from_shape_vec( + DecodingSampleBuffer::F32(d) => Ok(ArrayT::F32(Array2::from_shape_vec( (self.shape.y, self.shape.x), d, )?)), - DecodingResult::F64(d) => Ok(ArrayT::F64(Array2::from_shape_vec( + DecodingSampleBuffer::F64(d) => Ok(ArrayT::F64(Array2::from_shape_vec( (self.shape.y, self.shape.x), d, )?)), diff --git a/src/stats.rs b/ndbioimage/src/stats.rs similarity index 100% rename from src/stats.rs rename to ndbioimage/src/stats.rs diff --git a/src/tiffwrite.rs b/ndbioimage/src/tiffwrite.rs similarity index 88% rename from src/tiffwrite.rs rename to ndbioimage/src/tiffwrite.rs index 66a18f0..395b162 100644 --- a/src/tiffwrite.rs +++ b/ndbioimage/src/tiffwrite.rs @@ -321,4 +321,44 @@ mod tests { batch_to_tiff(&files_in, &files_out, None, None, true, true, None)?; Ok(()) } + + /// a rewritten file must not be served from the process-wide cache + #[cfg(feature = "tiff")] + #[test] + fn cache_invalidated_after_rewrite() -> Result<(), Error> { + use crate::view::View; + use ndarray::Array2; + use std::time::{Duration, SystemTime}; + use tiffwrite::IJTiffFile; + + let dir = std::env::temp_dir().join(format!("ndbioimage-rewrite-{}", std::process::id())); + create_dir_all(&dir)?; + let path = dir.join("cache_rewrite.tif"); + + let write = |value: u16| -> Result<(), Error> { + let mut tiff = IJTiffFile::new(&path)?; + tiff.save(Array2::::from_elem((4, 4), value), 0, 0, 0)?; + Ok(()) + }; + + write(1)?; + let view: View<_, DynReader> = View::from_path(&path)?; + let frame: Array2 = view.get_frame(0, 0, 0)?; + assert_eq!(frame[(0, 0)], 1); + + write(2)?; + // make the modification time unambiguous even on filesystems with a + // coarse timestamp resolution + let file = std::fs::OpenOptions::new().write(true).open(&path)?; + file.set_modified(SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000_000))?; + drop(file); + + let view: View<_, DynReader> = View::from_path(&path)?; + let frame: Array2 = view.get_frame(0, 0, 0)?; + assert_eq!(frame[(0, 0)], 2); + + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_dir(&dir); + Ok(()) + } } diff --git a/src/transforms.rs b/ndbioimage/src/transforms.rs similarity index 90% rename from src/transforms.rs rename to ndbioimage/src/transforms.rs index 498b8c4..cf6e975 100644 --- a/src/transforms.rs +++ b/ndbioimage/src/transforms.rs @@ -46,14 +46,49 @@ impl TryFrom> for Transforms { } } -#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] pub struct Transforms { - #[serde(skip_serializing_if = "HashMap::is_empty", default = "HashMap::new")] + #[serde(default)] pub channel: HashMap>, - #[serde(skip_serializing_if = "Vec::is_empty", default = "Vec::new")] + #[serde(default)] pub drift: Vec>, } +impl Serialize for Transforms { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + #[derive(Serialize)] + struct HumanReadable<'a> { + #[serde(skip_serializing_if = "HashMap::is_empty")] + channel: &'a HashMap>, + #[serde(skip_serializing_if = "Vec::is_empty")] + drift: &'a Vec>, + } + + #[derive(Serialize)] + struct Binary<'a> { + channel: &'a HashMap>, + drift: &'a Vec>, + } + + if serializer.is_human_readable() { + HumanReadable { + channel: &self.channel, + drift: &self.drift, + } + .serialize(serializer) + } else { + Binary { + channel: &self.channel, + drift: &self.drift, + } + .serialize(serializer) + } + } +} + impl Transforms { pub fn load>(path: P) -> Result { let mut file = std::fs::File::open(&path)?; diff --git a/src/utils.rs b/ndbioimage/src/utils.rs similarity index 100% rename from src/utils.rs rename to ndbioimage/src/utils.rs diff --git a/src/view.rs b/ndbioimage/src/view.rs similarity index 98% rename from src/view.rs rename to ndbioimage/src/view.rs index a9fb055..eb0ebb3 100644 --- a/src/view.rs +++ b/ndbioimage/src/view.rs @@ -1,5 +1,5 @@ use crate::axes::{Ax, Axis, Operation, Shape, Slice, SliceInfoElemDef, slice_info}; -use crate::cache::{ArrayCache, ArrayKeyRef, FrameCache, FrameKeyRef, ReaderKeyRef}; +use crate::cache::{ArrayCache, ArrayKeyRef, FileStamp, FrameCache, FrameKeyRef, ReaderKeyRef}; use crate::error::Error; use crate::metadata::Metadata; use crate::readers::{Dimensions, DynReader, Frame, Reader}; @@ -274,7 +274,7 @@ impl View { &self.axes } - #[expect(dead_code)] + #[allow(dead_code)] pub(crate) fn get_operations(&self) -> &IndexMap { &self.operations } @@ -964,6 +964,7 @@ impl View { path: self.reader.path(), series: self.reader.series(), position: self.reader.position(), + stamp: FileStamp::from_path(self.reader.path()), }, c, z, @@ -978,6 +979,7 @@ impl View { path: self.reader.path(), series: self.reader.series(), position: self.reader.position(), + stamp: FileStamp::from_path(self.reader.path()), }, dtype: type_name::(), slice: &self.slice, @@ -1226,14 +1228,17 @@ to_bytes_vec_impl!( #[cfg(test)] mod tests { use crate::axes::{Axis, Operation}; - use crate::cache::{ArrayCache, ArrayKey, ArrayKeyRef, FrameCache, ReaderKey, ReaderKeyRef}; + use crate::cache::{ + ArrayCache, ArrayKey, ArrayKeyRef, FileStamp, FrameCache, ReaderKey, ReaderKeyRef, + }; use crate::error::Error; use crate::readers::{DynReader, Frame, Reader}; use crate::stats::MinMax; - use crate::view::Item; + use crate::view::{Item, View}; use indexmap::IndexMap; - use ndarray::{Array, Array4, Array5, NewAxis}; + use ndarray::{Array, Array4, Array5, Ix5, NewAxis}; use ndarray::{Array2, ArrayD, IxDyn, SliceInfoElem, s}; + use postcard::{from_bytes, to_stdvec}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -1514,6 +1519,7 @@ mod tests { path: PathBuf::from("test.tif"), series: 0, position: 0, + stamp: FileStamp::default(), }, c, z, @@ -1549,6 +1555,7 @@ mod tests { path: PathBuf::from("test.tif"), series: 0, position: 0, + stamp: FileStamp::default(), }, dtype, slice: vec![SliceInfoElem::Index(index)], @@ -1574,6 +1581,7 @@ mod tests { path: Path::new("test.tif"), series: 0, position: 0, + stamp: FileStamp::default(), }, dtype: "u16", slice: &[SliceInfoElem::Index(0)], @@ -1604,4 +1612,18 @@ mod tests { assert_eq!(a, c2); Ok(()) } + + #[test] + fn serde() -> Result<(), Error> { + let file = "tiff/20251014_20-Pos_000_000_max.tif"; + let reader = open(file)?; + let view = reader.view(); + let bytes = to_stdvec(&view)?; + let view2: View = from_bytes(&bytes)?; + + println!("{:?}", view); + println!("{:?}", view2); + + Ok(()) + } } diff --git a/tests/test_open.py b/ndbioimage/tests/test_open.py similarity index 100% rename from tests/test_open.py rename to ndbioimage/tests/test_open.py diff --git a/tests/test_slicing.py b/ndbioimage/tests/test_slicing.py similarity index 100% rename from tests/test_slicing.py rename to ndbioimage/tests/test_slicing.py diff --git a/tests/test_ufuncs.py b/ndbioimage/tests/test_ufuncs.py similarity index 100% rename from tests/test_ufuncs.py rename to ndbioimage/tests/test_ufuncs.py diff --git a/py/ndbioimage/__init__.pyi b/py/ndbioimage/__init__.pyi deleted file mode 100644 index 43c5441..0000000 --- a/py/ndbioimage/__init__.pyi +++ /dev/null @@ -1,3 +0,0 @@ -from .ndbioimage_rs import Imread, Shape - -__all__ = ["Imread", "Shape"]