Skip to content

StarDistPanoptic

StarDistPanoptic

Bases: BaseModelPanoptic

Source code in src/histolytics/models/stardist_panoptic.py
class StarDistPanoptic(BaseModelPanoptic):
    model_name = "stardist_panoptic"

    def __init__(
        self,
        n_nuc_classes: int,
        n_tissue_classes: int,
        n_rays: int = 32,
        enc_name: str = "efficientnet_b5",
        enc_pretrain: bool = True,
        enc_freeze: bool = False,
        device: torch.device = torch.device("cuda"),
        model_kwargs: Dict[str, Any] = {},
    ) -> None:
        """StardistPanoptic model for panoptic segmentation of nuclei and tissues.

        Note:
            [Stardist article](https://arxiv.org/abs/1806.03535)

        Parameters:
            n_nuc_classes (int):
                Number of nuclei type classes.
            n_tissue_classes (int):
                Number of tissue type classes.
            n_rays (int):
                Number of rays for the Stardist model.
            enc_name (str):
                Name of the pytorch-image-models encoder.
            enc_pretrain (bool):
                Whether to use pretrained weights in the encoder.
            enc_freeze (bool):
                Freeze encoder weights for training.
            device (torch.device):
                Device to run the model on.
            model_kwargs (dict):
                Additional keyword arguments for the model.
        """
        super().__init__()
        self.model = stardist_panoptic(
            n_rays=n_rays,
            n_nuc_classes=n_nuc_classes,
            n_tissue_classes=n_tissue_classes,
            enc_name=enc_name,
            enc_pretrain=enc_pretrain,
            enc_freeze=enc_freeze,
            **model_kwargs,
        )

        self.device = device
        self.model.to(device)

    def set_inference_mode(
        self,
        mixed_precision: bool = True,
        postproc_kwargs: Dict[str, Any] = {"trim_bboxes": True},
    ) -> None:
        """Set model to inference mode."""
        self.model.eval()
        self.predictor = Predictor(
            model=self.model,
            mixed_precision=mixed_precision,
        )
        self.post_processor = PostProcessor(
            postproc_method="stardist",
            postproc_kwargs=postproc_kwargs,
        )
        self.inference_mode = True

set_inference_mode

set_inference_mode(
    mixed_precision: bool = True,
    postproc_kwargs: Dict[str, Any] = {"trim_bboxes": True},
) -> None

Set model to inference mode.

Source code in src/histolytics/models/stardist_panoptic.py
def set_inference_mode(
    self,
    mixed_precision: bool = True,
    postproc_kwargs: Dict[str, Any] = {"trim_bboxes": True},
) -> None:
    """Set model to inference mode."""
    self.model.eval()
    self.predictor = Predictor(
        model=self.model,
        mixed_precision=mixed_precision,
    )
    self.post_processor = PostProcessor(
        postproc_method="stardist",
        postproc_kwargs=postproc_kwargs,
    )
    self.inference_mode = True

from_pretrained classmethod

from_pretrained(
    weights: Union[str, Path],
    device: device = torch.device("cuda"),
    model_kwargs: Dict[str, Any] = {},
) -> BaseModelPanoptic

Load the model from pretrained weights.

Parameters:

Name Type Description Default
weights Union[str, Path]

Registered checkpoint name or local checkpoint path.

required
device device

Device to load and run the model on. Defaults to CUDA.

device('cuda')
model_kwargs Dict[str, Any]

Additional arguments for the model.

{}

Returns:

Type Description
BaseModelPanoptic

A model with strictly loaded weights. Registered checkpoints provide

BaseModelPanoptic

independent nuclei/tissue class dictionaries on the returned instance;

BaseModelPanoptic

local checkpoints have None mappings. Loading or customizing another

BaseModelPanoptic

model does not change these dictionaries.

Raises:

Type Description
ValueError

If weights is neither a file nor a registered name. Checkpoint loading and model construction errors propagate.

Examples:

>>> model = Model.from_pretrained(<str or Path to weights>, device=torch.device("cuda"))
Source code in src/histolytics/models/_base_model.py
@classmethod
def from_pretrained(
    cls,
    weights: Union[str, Path],
    device: torch.device = torch.device("cuda"),
    model_kwargs: Dict[str, Any] = {},
) -> "BaseModelPanoptic":
    """Load the model from pretrained weights.

    Args:
        weights: Registered checkpoint name or local checkpoint path.
        device: Device to load and run the model on. Defaults to CUDA.
        model_kwargs: Additional arguments for the model.

    Returns:
        A model with strictly loaded weights. Registered checkpoints provide
        independent nuclei/tissue class dictionaries on the returned instance;
        local checkpoints have None mappings. Loading or customizing another
        model does not change these dictionaries.

    Raises:
        ValueError: If weights is neither a file nor a registered name.
            Checkpoint loading and model construction errors propagate.

    Examples:
        >>> model = Model.from_pretrained(<str or Path to weights>, device=torch.device("cuda"))
    """
    weights_path = Path(weights)
    if not weights_path.is_file():
        if weights_path.as_posix() in PRETRAINED_MODELS[cls.model_name].keys():
            weights_path = Path(
                hf_hub_download(
                    repo_id=PRETRAINED_MODELS[cls.model_name][weights]["repo_id"],
                    filename=PRETRAINED_MODELS[cls.model_name][weights]["filename"],
                )
            )

        else:
            raise ValueError(
                "Please provide a valid path. or a pre-trained model downloaded from the"
                f" histolytics-hub. One of {list(PRETRAINED_MODELS[cls.model_name].keys())}."
            )

    enc_name, n_nuc_classes, n_tissue_classes, state_dict = cls._get_state_dict(
        weights_path, device=device
    )

    model_inst = cls(
        n_nuc_classes=n_nuc_classes,
        n_tissue_classes=n_tissue_classes,
        enc_name=enc_name,
        enc_pretrain=False,
        enc_freeze=False,
        device=device,
        model_kwargs=model_kwargs,
    )

    if weights_path.suffix == ".safetensors":
        try:
            from safetensors.torch import load_model
        except ImportError:
            raise ImportError(
                "Please install `safetensors` package to load .safetensors files."
            )
        load_model(model_inst.model, weights_path, device.type)
    else:
        model_inst.model.load_state_dict(state_dict, strict=True)

    try:
        model_inst.nuc_classes = MODEL_CLASS_DICTS[weights]["nuc"].copy()
        model_inst.tissue_classes = MODEL_CLASS_DICTS[weights]["tissue"].copy()
    except KeyError:
        # if the model is not in the class dict, set to None
        model_inst.nuc_classes = None
        model_inst.tissue_classes = None

    return model_inst

predict

predict(
    x: Union[Tensor, ndarray, Image],
    *,
    use_sliding_win: bool = False,
    window_size: Tuple[int, int] = None,
    stride: int = None,
    save_intermediate: bool = False,
) -> Dict[
    str, Union[SoftSemanticOutput, SoftInstanceOutput]
]

Predict the input image or image batch.

Parameters:

Name Type Description Default
x Union[Tensor, ndarray, Image]

Input image (H, W, C) or input image batch (B, C, H, W).

required
use_sliding_win bool

Whether to use sliding window for prediction.

False
window_size Tuple[int, int]

The height and width of the sliding window. If use_sliding_win is False this argument is ignored.

None
stride int

The stride for the sliding window. If use_sliding_win is False this argument is ignored.

None
save_intermediate bool, default=False

Whether to save intermediate results (logits). If True, the method returns a tuple (final predictions, intermediate results), where the intermediate results are the raw model outputs before argmax.

False

Returns:

Type Description
Dict[str, Union[SoftSemanticOutput, SoftInstanceOutput]]

Dict[str, Union[SoftSemanticOutput, SoftInstanceOutput]]: Dictionary of soft outputs:

- "nuclei": SoftInstanceOutput (type_map, aux_map).
- "tissue": SoftSemanticOutput (type_map).

Examples:

>>> my_model.set_inference_mode()
>>> # with sliding window if image is large
>>> x = my_model.predict(x=image, use_sliding_win=True, window_size=(256, 256), stride=128)
>>> # without sliding window if image is small enough
>>> x = my_model.predict(x=image, use_sliding_win=False)
Source code in src/histolytics/models/_base_model.py
def predict(
    self,
    x: Union[torch.Tensor, np.ndarray, Image],
    *,
    use_sliding_win: bool = False,
    window_size: Tuple[int, int] = None,
    stride: int = None,
    save_intermediate: bool = False,
) -> Dict[str, Union[SoftSemanticOutput, SoftInstanceOutput]]:
    """Predict the input image or image batch.

    Parameters:
        x (Union[torch.Tensor, np.ndarray, Image]):
            Input image (H, W, C) or input image batch (B, C, H, W).
        use_sliding_win (bool):
            Whether to use sliding window for prediction.
        window_size (Tuple[int, int]):
            The height and width of the sliding window. If `use_sliding_win` is False
            this argument is ignored.
        stride (int):
            The stride for the sliding window. If `use_sliding_win` is False this
            argument is ignored.
        save_intermediate (bool, default=False):
            Whether to save intermediate results (logits). If True, the method
            returns a tuple (final predictions, intermediate results), where the
            intermediate results are the raw model outputs before argmax.

    Returns:
        Dict[str, Union[SoftSemanticOutput, SoftInstanceOutput]]:
            Dictionary of soft outputs:

                - "nuclei": SoftInstanceOutput (type_map, aux_map).
                - "tissue": SoftSemanticOutput (type_map).

    Examples:
        >>> my_model.set_inference_mode()
        >>> # with sliding window if image is large
        >>> x = my_model.predict(x=image, use_sliding_win=True, window_size=(256, 256), stride=128)
        >>> # without sliding window if image is small enough
        >>> x = my_model.predict(x=image, use_sliding_win=False)
    """
    if not self.inference_mode:
        raise ValueError("Run `.set_inference_mode()` before running `predict`")

    if not use_sliding_win:
        x = self.predictor.predict(
            x=x, apply_boundary_weight=False, save_intermediate=save_intermediate
        )
    else:
        if window_size is None:
            raise ValueError(
                "`window_size` must be provided when using sliding window."
            )
        if stride is None:
            raise ValueError("`stride` must be provided when using sliding window.")

        x = self.predictor.predict_sliding_win(
            x=x,
            window_size=window_size,
            stride=stride,
            apply_boundary_weight=True,
            save_intermediate=save_intermediate,
        )

    return x

post_process

post_process(
    x: dict[
        str, SoftSemanticOutput | SoftInstanceOutput | None
    ],
    *,
    use_async_postproc: bool = True,
    start_method: str = "threading",
    n_jobs: int = 4,
    save_paths_nuc: list[Path | str] | None = None,
    save_paths_cyto: list[Path | str] | None = None,
    save_paths_tissue: list[Path | str] | None = None,
    coords: list[tuple[int, int, int, int]] | None = None,
    class_dict_nuc: dict[int, str] | None = None,
    class_dict_cyto: dict[int, str] | None = None,
    class_dict_tissue: dict[int, str] | None = None,
    nuc_smooth_func: Callable | None = gaussian_smooth,
    cyto_smooth_func: Callable | None = gaussian_smooth,
    tissue_smooth_func: Callable | None = None,
) -> dict[str, list[Any] | None]

Reconstruct labels from predictor outputs and optionally save geometry.

Parameters:

Name Type Description Default
x dict[str, SoftSemanticOutput | SoftInstanceOutput | None]

Predictor output with "nuc", "cyto", and "tissue" keys. This model facade requires nuclei and tissue maps; cytoplasm may be None. Type maps contain class IDs with shape (B, H, W), not logits. Instance auxiliary maps have shape (B, C_aux, H, W), where C_aux and values depend on the reconstruction algorithm. Optional binary maps contain foreground probabilities with shape (B, 1, H, W). The upstream processor handles tensor detachment and CPU conversion.

required
use_async_postproc bool

Use asynchronous processing for batches larger than one. Single-image batches always run serially.

True
start_method str

Worker backend: "threading", "fork", or "spawn".

'threading'
n_jobs int

Number of workers for parallel processing.

4
save_paths_nuc list[Path | str] | None

One destination per image for nuclei geometry, or None to return reconstructed nuclei arrays.

None
save_paths_cyto list[Path | str] | None

Cytoplasm destinations, or None to return arrays.

None
save_paths_tissue list[Path | str] | None

Tissue destinations, or None to return arrays.

None
coords list[tuple[int, int, int, int]] | None

Per-image (x, y, width, height) in pixels. Vector output is translated by x and y; no physical-unit conversion is performed.

None
class_dict_nuc dict[int, str] | None

Nuclei class-ID to class-name mapping for vector output.

None
class_dict_cyto dict[int, str] | None

Cytoplasm class mapping for vector output.

None
class_dict_tissue dict[int, str] | None

Tissue class mapping for vector output.

None
nuc_smooth_func Callable | None

Geometry callback for saved nuclei polygons. None disables smoothing; ignored for in-memory array outputs. Async callbacks must be pickleable, for example module-level functions.

gaussian_smooth
cyto_smooth_func Callable | None

Geometry callback for saved cytoplasm polygons.

gaussian_smooth
tissue_smooth_func Callable | None

Geometry callback for saved tissue polygons.

None

Returns:

Type Description
dict[str, list[Any] | None]

The upstream processor result, unchanged, under "nuc", "cyto", and

dict[str, list[Any] | None]

"tissue". In serial/synchronous processing, instance results are lists

dict[str, list[Any] | None]

of (instance labels, class labels) int32 arrays with shape (H, W), and

dict[str, list[Any] | None]

tissue results are lists of int32 class-label arrays. Absent outputs

dict[str, list[Any] | None]

are None; saved entries are None instead of arrays. The published

dict[str, list[Any] | None]

asynchronous processor's cytoplasm return is not normalized here.

Raises:

Type Description
ValueError

If inference mode has not been enabled. Processor and file-writing errors propagate to the caller.

Examples:

>>> my_model.set_inference_mode()
>>> x = my_model.predict(x=image, use_sliding_win=False)
>>> x = my_model.post_process(
...     x,
...     use_async_postproc=True,
...     start_method="threading",
...     n_jobs=4,
... )
Source code in src/histolytics/models/_base_model.py
def post_process(
    self,
    x: dict[str, SoftSemanticOutput | SoftInstanceOutput | None],
    *,
    use_async_postproc: bool = True,
    start_method: str = "threading",
    n_jobs: int = 4,
    save_paths_nuc: list[Path | str] | None = None,
    save_paths_cyto: list[Path | str] | None = None,
    save_paths_tissue: list[Path | str] | None = None,
    coords: list[tuple[int, int, int, int]] | None = None,
    class_dict_nuc: dict[int, str] | None = None,
    class_dict_cyto: dict[int, str] | None = None,
    class_dict_tissue: dict[int, str] | None = None,
    nuc_smooth_func: Callable | None = gaussian_smooth,
    cyto_smooth_func: Callable | None = gaussian_smooth,
    tissue_smooth_func: Callable | None = None,
) -> dict[str, list[Any] | None]:
    """Reconstruct labels from predictor outputs and optionally save geometry.

    Args:
        x: Predictor output with "nuc", "cyto", and "tissue" keys. This model
            facade requires nuclei and tissue maps; cytoplasm may be None.
            Type maps contain class IDs with shape (B, H, W), not logits.
            Instance auxiliary maps have shape (B, C_aux, H, W), where
            C_aux and values depend on the reconstruction algorithm. Optional
            binary maps contain foreground probabilities with shape (B, 1, H, W).
            The upstream processor handles tensor detachment and CPU conversion.
        use_async_postproc: Use asynchronous processing for batches larger
            than one. Single-image batches always run serially.
        start_method: Worker backend: "threading", "fork", or "spawn".
        n_jobs: Number of workers for parallel processing.
        save_paths_nuc: One destination per image for nuclei geometry, or None
            to return reconstructed nuclei arrays.
        save_paths_cyto: Cytoplasm destinations, or None to return arrays.
        save_paths_tissue: Tissue destinations, or None to return arrays.
        coords: Per-image (x, y, width, height) in pixels. Vector output is
            translated by x and y; no physical-unit conversion is performed.
        class_dict_nuc: Nuclei class-ID to class-name mapping for vector output.
        class_dict_cyto: Cytoplasm class mapping for vector output.
        class_dict_tissue: Tissue class mapping for vector output.
        nuc_smooth_func: Geometry callback for saved nuclei polygons. None
            disables smoothing; ignored for in-memory array outputs. Async
            callbacks must be pickleable, for example module-level functions.
        cyto_smooth_func: Geometry callback for saved cytoplasm polygons.
        tissue_smooth_func: Geometry callback for saved tissue polygons.

    Returns:
        The upstream processor result, unchanged, under "nuc", "cyto", and
        "tissue". In serial/synchronous processing, instance results are lists
        of (instance labels, class labels) int32 arrays with shape (H, W), and
        tissue results are lists of int32 class-label arrays. Absent outputs
        are None; saved entries are None instead of arrays. The published
        asynchronous processor's cytoplasm return is not normalized here.

    Raises:
        ValueError: If inference mode has not been enabled. Processor and
            file-writing errors propagate to the caller.

    Examples:
        >>> my_model.set_inference_mode()
        >>> x = my_model.predict(x=image, use_sliding_win=False)
        >>> x = my_model.post_process(
        ...     x,
        ...     use_async_postproc=True,
        ...     start_method="threading",
        ...     n_jobs=4,
        ... )
    """
    if not self.inference_mode:
        raise ValueError(
            "Run `.set_inference_mode()` before running `post_process`"
        )

    # if batch size is 1, run serially
    if x["tissue"].type_map.shape[0] == 1:
        return self.post_processor.postproc_serial(
            x,
            save_paths_nuc=save_paths_nuc,
            save_paths_cyto=save_paths_cyto,
            save_paths_tissue=save_paths_tissue,
            coords=coords,
            class_dict_nuc=class_dict_nuc,
            class_dict_cyto=class_dict_cyto,
            class_dict_tissue=class_dict_tissue,
            nuc_smooth_func=nuc_smooth_func,
            cyto_smooth_func=cyto_smooth_func,
            tissue_smooth_func=tissue_smooth_func,
        )

    if use_async_postproc:
        x = self.post_processor.postproc_parallel_async(
            x,
            start_method=start_method,
            n_jobs=n_jobs,
            save_paths_nuc=save_paths_nuc,
            save_paths_cyto=save_paths_cyto,
            save_paths_tissue=save_paths_tissue,
            coords=coords,
            class_dict_nuc=class_dict_nuc,
            class_dict_cyto=class_dict_cyto,
            class_dict_tissue=class_dict_tissue,
            nuc_smooth_func=nuc_smooth_func,
            cyto_smooth_func=cyto_smooth_func,
            tissue_smooth_func=tissue_smooth_func,
        )
    else:
        x = self.post_processor.postproc_parallel(
            x,
            start_method=start_method,
            n_jobs=n_jobs,
            save_paths_nuc=save_paths_nuc,
            save_paths_cyto=save_paths_cyto,
            save_paths_tissue=save_paths_tissue,
            coords=coords,
            class_dict_nuc=class_dict_nuc,
            class_dict_cyto=class_dict_cyto,
            class_dict_tissue=class_dict_tissue,
            nuc_smooth_func=nuc_smooth_func,
            cyto_smooth_func=cyto_smooth_func,
            tissue_smooth_func=tissue_smooth_func,
        )

    return x