Merge branch 'release/4.0.6'

This commit is contained in:
Hervé BREDIN committed 2026-06-29 15:23:43 +02:00
commit b0111d10d2
4 files changed
+541 -55

No files matched your search

+4
View File
@@ -1,5 +1,9 @@
# CHANGELOG
## next
- improve(pipeline): add support for `subfolder` in `Pipeline.from_pretrained`
## Version 4.0.5 (2026-06-22)
- improve(telemetry): reduce number of sent packages [@litdarya](https://github.com/litdarya)
+17 -20
View File
@@ -4,7 +4,7 @@
# MIT License
#
# Copyright (c) 2024-2025 CNRS
# Copyright (c) 2025 pyannoteAI
# Copyright (c) 2025- pyannoteAI
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -97,6 +97,7 @@ def parse_device(device: Device) -> torch.device:
def get_diarization(prediction) -> Annotation:
# if result is an Annotation, assume it is speaker diarization
if isinstance(prediction, Annotation):
return prediction
@@ -603,7 +604,10 @@ def benchmark(
# load pretrained pipeline
pretrained_pipeline = Pipeline.from_pretrained(
pipeline, revision=revision, token=token, cache_dir=cache,
pipeline,
revision=revision,
token=token,
cache_dir=cache,
)
if pretrained_pipeline is None:
print(f"Could not load pretrained pipeline from {pipeline}.")
@@ -646,10 +650,6 @@ def benchmark(
if num_speakers == NumSpeakers.ORACLE:
benchmark_name += ".OracleNumSpeakers"
# used to store processing time and file duration
processing_time: dict[str, float] = dict()
playing_time: dict[str, float] = dict()
# used to store raw predictions in JSON format
serialized_predictions: dict[str, dict] = dict()
@@ -676,19 +676,14 @@ def benchmark(
if rttm_file.exists():
raise FileExistsError(f"{rttm_file} already exists.")
# iterate over all files in the specified subset
for file in track(files, disable=not progress):
# gather file metadata
uri: str = file["uri"]
playing_time[uri] = Audio().get_duration(file)
if hasattr(pretrained_pipeline, "apply_batch"):
iterator = pretrained_pipeline(files, progress=progress)
else:
iterator = track(pretrained_pipeline(files), disable=not progress)
tic: float = time.time()
# apply pretrained pipeline to file
prediction = pretrained_pipeline(file, **file.get("pipeline_kwargs", {}))
tac: float = time.time()
processing_time[uri] = tac - tic
tic: float = time.time()
for file, prediction in iterator:
uri = file["uri"]
# if prediction has a built-in serialize method, save serialized version
if hasattr(prediction, "serialize"):
@@ -731,6 +726,8 @@ def benchmark(
if optimize:
file["speaker_diarization"] = speaker_diarization
tac: float = time.time()
# save serialized predictions to disk (might contain more than just diarization results)
if serialized_predictions and not per_file:
with open(into / f"{benchmark_name}.json", "w") as f:
@@ -738,8 +735,8 @@ def benchmark(
# log processing time and capacity
processing = dict()
total_processing_time: float = sum(processing_time.values())
total_playing_time: float = sum(playing_time.values())
total_processing_time = tac - tic
total_playing_time = sum(Audio().get_duration(file) for file in files)
processing["seconds_per_hour"] = total_processing_time / (total_playing_time / 3600)
processing["times_faster_than_realtime"] = (
total_playing_time / total_processing_time
+149 -35
View File
@@ -22,13 +22,14 @@
# SOFTWARE.
from __future__ import annotations
import os
import warnings
from collections import OrderedDict
from collections.abc import Iterator
from functools import partial
from pathlib import Path
from typing import Callable, Dict, List, Optional
from typing import Any, Callable, Dict, List, Mapping, Optional
import torch
import torch.nn as nn
@@ -49,6 +50,7 @@ from pyannote.pipeline import Pipeline as _Pipeline
def expand_subfolders(
config,
model_id: str | None = None,
parent_subfolder: str | None = None,
parent_revision: str | None = None,
cache_dir: Path | str | None = None,
token: str | None = None,
@@ -65,6 +67,9 @@ def expand_subfolders(
config : dict
model_id : str, optional
Model identifier when loading from the huggingface.co model hub.
parent_subfolder : str, optional
Subfolder from which the pipeline config was loaded. When set, child
"$model/{child}" paths are resolved as "{parent_subfolder}/{child}".
parent_revision : str, optional
Revision when loading from the huggingface.co model hub.
token : str or bool, optional
@@ -85,6 +90,11 @@ def expand_subfolders(
else:
revision = parent_revision
if parent_subfolder:
subfolder = (
f"{parent_subfolder.rstrip('/')}/{subfolder.lstrip('/')}"
)
config[key] = {
"checkpoint": model_id,
"revision": revision,
@@ -96,6 +106,7 @@ def expand_subfolders(
expand_subfolders(
value,
model_id,
parent_subfolder=parent_subfolder,
parent_revision=parent_revision,
token=token,
cache_dir=cache_dir,
@@ -113,6 +124,11 @@ def expand_subfolders(
else:
revision = parent_revision
if parent_subfolder:
subfolder = (
f"{parent_subfolder.rstrip('/')}/{subfolder.lstrip('/')}"
)
config[idx] = {
"checkpoint": model_id,
"revision": revision,
@@ -125,6 +141,7 @@ def expand_subfolders(
expand_subfolders(
value,
model_id,
parent_subfolder=parent_subfolder,
parent_revision=parent_revision,
token=token,
cache_dir=cache_dir,
@@ -138,6 +155,7 @@ class Pipeline(_Pipeline):
checkpoint: str | Path | dict,
revision: str | None = None,
hparams_file: str | Path | None = None,
subfolder: str | None = None,
token: str | bool | None = None,
cache_dir: Path | str | None = None,
) -> Optional["Pipeline"]:
@@ -154,6 +172,8 @@ class Pipeline(_Pipeline):
revision : str, optional
Revision when loading from the huggingface.co model hub.
hparams_file: Path or str, optional
subfolder : str, optional
Folder inside the hf.co model repo (or local directory).
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
@@ -165,23 +185,34 @@ class Pipeline(_Pipeline):
if isinstance(checkpoint, dict):
if revision is not None:
raise ValueError("Revisions cannot be used with local checkpoints.")
if subfolder is not None:
raise ValueError(
"Subfolder cannot be used when checkpoint is a config dictionary. "
)
model_id = Path.cwd()
config = checkpoint
otel_origin: str = "local"
# if checkpoint is a directory, look for the pipeline checkpoint
# inside this directory
# inside this directory (or inside a subfolder if specified)
elif os.path.isdir(checkpoint):
if revision is not None:
raise ValueError("Revisions cannot be used with local checkpoints.")
model_id = Path(checkpoint)
config_yml = model_id / AssetFileName.Pipeline.value
if subfolder:
config_yml = model_id / subfolder / AssetFileName.Pipeline.value
else:
config_yml = model_id / AssetFileName.Pipeline.value
otel_origin: str = "local"
# if checkpoint is a file, assume it is the pipeline checkpoint
elif os.path.isfile(checkpoint):
if revision is not None:
raise ValueError("Revisions cannot be used with local checkpoints.")
if subfolder is not None:
raise ValueError(
"Subfolder cannot be used when checkpoint is a path to a config.yaml file. "
)
model_id = Path(checkpoint).parent
config_yml = checkpoint
otel_origin: str = "local"
@@ -198,6 +229,7 @@ class Pipeline(_Pipeline):
config_yml = download_from_hf_hub(
model_id,
AssetFileName.Pipeline,
subfolder=subfolder,
revision=revision,
cache_dir=cache_dir,
token=token,
@@ -219,6 +251,7 @@ class Pipeline(_Pipeline):
expand_subfolders(
config,
model_id,
parent_subfolder=subfolder,
parent_revision=revision,
token=token,
cache_dir=cache_dir,
@@ -407,45 +440,25 @@ class Pipeline(_Pipeline):
"""
raise NotImplementedError()
def __call__(self, file: AudioFile, preload: bool = False, **kwargs):
"""Validate file, (optionally) load it in memory, then process it
def prepare_one(self, file: AudioFile, preload: bool = False) -> Mapping:
"""Prepare one file for being processed by the pipeline
1. Validate file
2. Add pipeline preprocessors if any
3. Preload in memory if requested
Parameters
----------
file : AudioFile
File to process
preload : bool, optional
Whether to preload waveform before applying the pipeline.
kwargs : keyword arguments, optional
Additional keyword arguments passed to `self.apply(...)`
preload : bool
Whether to preload in memory
Returns
-------
output : Any
Whatever `self.apply(...)` returns
file : Mapping
Prepared file
"""
fix_reproducibility(getattr(self, "device", torch.device("cpu")))
if not self.instantiated:
# instantiate with default parameters when available
try:
default_parameters = self.default_parameters()
except NotImplementedError:
raise RuntimeError(
"A pipeline must be instantiated with `pipeline.instantiate(parameters)` before it can be applied."
)
try:
self.instantiate(default_parameters)
except ValueError:
raise RuntimeError(
"A pipeline must be instantiated with `pipeline.instantiate(parameters)` before it can be applied. "
"Tried to use parameters provided by `pipeline.default_parameters()` but those are not compatible. "
)
warnings.warn(
f"The pipeline has been automatically instantiated with {default_parameters}."
)
file = Audio.validate_file(file)
@@ -471,11 +484,110 @@ class Pipeline(_Pipeline):
# therefore we remove the `channel` key from the file
file.pop("channel", None)
return file
def __call__(
self,
file: AudioFile | list[AudioFile],
preload: bool = False,
**kwargs,
) -> Any | Iterator[tuple[str, Any]]:
"""Validate file(s), (optionally) load it/them in memory, then process it/them
Parameters
----------
file : AudioFile or list[AudioFile]
File to process, or a list of AudioFiles for batch processing.
All files in the list must have distinct URIs.
Files are processed in sequence unless the pipeline defines an
`apply_batch` method, which is called instead.
preload : bool, optional
Whether to preload waveform before applying the pipeline.
Ignored for batch input when `apply_batch` is available.
kwargs : keyword arguments, optional
Additional keyword arguments passed to `self.apply(...)` or
`self.apply_batch(...)`
Returns (for single file)
-------------------------
output : Any
Whatever `self.apply(...)` returns for a single file, or a dict
mapping each URI to its output for batch input.
Yields (for multiple files)
---------------------------
file : AudioFile
output : Any
"""
fix_reproducibility(getattr(self, "device", torch.device("cpu")))
if not self.instantiated:
# instantiate with default parameters when available
try:
default_parameters = self.default_parameters()
except NotImplementedError:
raise RuntimeError(
"A pipeline must be instantiated with `pipeline.instantiate(parameters)` before it can be applied."
)
try:
self.instantiate(default_parameters)
except ValueError:
raise RuntimeError(
"A pipeline must be instantiated with `pipeline.instantiate(parameters)` before it can be applied. "
"Tried to use parameters provided by `pipeline.default_parameters()` but those are not compatible. "
)
warnings.warn(
f"The pipeline has been automatically instantiated with {default_parameters}."
)
if isinstance(file, list):
# prepare every file
files = [self.prepare_one(f, preload=preload) for f in file]
# check for duplicate uris (and complain if any)
uris = [f["uri"] for f in files]
if len(uris) != len(set(uris)):
seen: set[str] = set()
duplicates = [u for u in uris if u in seen or seen.add(u)]
raise ValueError(
f"All files in a batch must have distinct URIs. "
f"Duplicate URIs: {duplicates}"
)
# if pipeline natively supports batch processing, use it
if hasattr(self, "apply_batch"):
# TODO: warn user that pipeline_kwargs might not be taken into account.
for f, prediction in self.apply_batch(files, **kwargs):
# send file duration to telemetry as well as
# requested number of speakers in case of diarization
track_pipeline_apply(self, f, **kwargs)
yield f, prediction
# otherwise process files sequentially
else:
for f in files:
prediction = self.apply(f, **f.get("pipeline_kwargs", {}), **kwargs)
# send file duration to telemetry as well as
# requested number of speakers in case of diarization
track_pipeline_apply(self, f, **kwargs)
yield f, prediction
return
file = self.prepare_one(file, preload=preload)
prediction = self.apply(file, **file.get("pipeline_kwargs", {}), **kwargs)
# send file duration to telemetry as well as
# requested number of speakers in case of diarization
track_pipeline_apply(self, file, **kwargs)
return self.apply(file, **kwargs)
return prediction
def to(self, device: torch.device) -> Pipeline:
"""Send pipeline to `device`"""
@@ -507,5 +619,7 @@ class Pipeline(_Pipeline):
return self.to(torch.device("cuda", device))
else:
if device.type != "cuda":
raise ValueError("Expected CUDA device. Use `Pipeline.to(device)` for other devices.")
raise ValueError(
"Expected CUDA device. Use `Pipeline.to(device)` for other devices."
)
return self.to(device)
+371
View File
@@ -0,0 +1,371 @@
# MIT License
#
# Copyright (c) 2025- pyannoteAI
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""Tests for subfolder support in Pipeline.from_pretrained"""
import pytest
import yaml
from pathlib import Path
from pyannote.audio.core.pipeline import Pipeline, expand_subfolders
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def write_config(directory: Path, config: dict, subfolder: str | None = None) -> Path:
"""Write a config.yaml inside *directory* (or *directory/subfolder*)."""
target = directory / subfolder if subfolder else directory
target.mkdir(parents=True, exist_ok=True)
cfg_path = target / "config.yaml"
cfg_path.write_text(yaml.dump(config))
return cfg_path
# from_pretrained always injects `token` and `cache_dir` into params via
# setdefault() before calling Klass(**params), so the subclass __init__ must
# accept (and ignore) them.
class _DummyPipeline(Pipeline):
"""Minimal concrete subclass of Pipeline for testing from_pretrained."""
def __init__(self, token=None, cache_dir=None, **kwargs):
super().__init__()
def apply(self, file, **kwargs):
return None
_DUMMY_CLS_DOTPATH = f"{_DummyPipeline.__module__}.{_DummyPipeline.__qualname__}"
class _DummyParentPipeline(Pipeline):
"""Pipeline that owns a single subpipeline loaded from a checkpoint dict."""
def __init__(self, sub=None, token=None, cache_dir=None):
super().__init__()
if isinstance(sub, dict):
# Assigning a Pipeline instance triggers __setattr__ which registers
# it in self._pipelines — the standard pyannote sub-pipeline pattern.
self.sub = Pipeline.from_pretrained(**sub)
def apply(self, file, **kwargs):
return None
_DUMMY_PARENT_CLS_DOTPATH = (
f"{_DummyParentPipeline.__module__}.{_DummyParentPipeline.__qualname__}"
)
# ---------------------------------------------------------------------------
# expand_subfolders unit tests
# ---------------------------------------------------------------------------
class TestExpandSubfolders:
"""expand_subfolders resolves $model/... references in config dicts/lists."""
def test_no_model_references(self):
config = {"key": "plain_value", "nested": {"k": 42}}
expand_subfolders(config, model_id="org/repo")
assert config == {"key": "plain_value", "nested": {"k": 42}}
def test_dict_model_reference_no_parent_subfolder(self):
config = {"embedding": "$model/embeddings"}
expand_subfolders(config, model_id="org/repo", token="tok")
assert config["embedding"] == {
"checkpoint": "org/repo",
"revision": None,
"subfolder": "embeddings",
"token": "tok",
"cache_dir": None,
}
def test_dict_model_reference_with_parent_subfolder(self):
"""Child $model refs are resolved under parent_subfolder."""
config = {"segmentation": "$model/seg"}
expand_subfolders(
config,
model_id="org/repo",
parent_subfolder="v1",
token=None,
)
assert config["segmentation"]["subfolder"] == "v1/seg"
def test_dict_model_reference_with_explicit_revision(self):
"""@revision in $model/path@rev overrides parent_revision."""
config = {"model": "$model/weights@abc123"}
expand_subfolders(config, model_id="org/repo", parent_revision="main")
assert config["model"]["revision"] == "abc123"
assert config["model"]["subfolder"] == "weights"
def test_dict_model_reference_inherits_parent_revision(self):
config = {"model": "$model/weights"}
expand_subfolders(config, model_id="org/repo", parent_revision="v2")
assert config["model"]["revision"] == "v2"
def test_list_model_reference_no_parent_subfolder(self):
config = ["$model/part_a", "$model/part_b"]
expand_subfolders(config, model_id="org/repo")
assert config[0]["subfolder"] == "part_a"
assert config[1]["subfolder"] == "part_b"
def test_list_model_reference_with_parent_subfolder(self):
config = ["$model/seg", "plain"]
expand_subfolders(config, model_id="org/repo", parent_subfolder="root")
assert config[0]["subfolder"] == "root/seg"
assert config[1] == "plain"
def test_nested_dict_model_references(self):
"""Recursion into nested dicts."""
config = {"outer": {"inner": "$model/deep"}}
expand_subfolders(config, model_id="org/repo", parent_subfolder="base")
assert config["outer"]["inner"]["subfolder"] == "base/deep"
def test_nested_list_model_references(self):
"""Recursion into lists that contain dicts."""
config = {"items": [{"model": "$model/a"}, {"model": "$model/b"}]}
expand_subfolders(config, model_id="org/repo", parent_subfolder="p")
assert config["items"][0]["model"]["subfolder"] == "p/a"
assert config["items"][1]["model"]["subfolder"] == "p/b"
def test_no_model_id_leaves_checkpoint_as_none(self):
"""When model_id is None the checkpoint field is also None."""
config = {"seg": "$model/seg"}
expand_subfolders(config, model_id=None)
assert config["seg"]["checkpoint"] is None
def test_parent_subfolder_combined_with_explicit_revision(self):
"""parent_subfolder and explicit @revision can coexist."""
config = {"model": "$model/weights@pinned"}
expand_subfolders(
config,
model_id="org/repo",
parent_subfolder="folder",
parent_revision="ignored",
)
assert config["model"]["subfolder"] == "folder/weights"
assert config["model"]["revision"] == "pinned"
# ---------------------------------------------------------------------------
# Pipeline.from_pretrained local-directory tests
# ---------------------------------------------------------------------------
class TestFromPretrainedLocalSubfolder:
"""Pipeline.from_pretrained with local directories and the subfolder arg."""
def _minimal_config(self) -> dict:
return {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}
def test_loads_config_from_root_without_subfolder(self, tmp_path):
"""Without subfolder, config.yaml is read from the directory root."""
write_config(tmp_path, self._minimal_config())
pipeline = Pipeline.from_pretrained(str(tmp_path))
assert isinstance(pipeline, _DummyPipeline)
def test_loads_config_from_subfolder(self, tmp_path):
"""With subfolder='sub', config.yaml is read from <root>/sub/config.yaml."""
write_config(tmp_path, self._minimal_config(), subfolder="sub")
pipeline = Pipeline.from_pretrained(str(tmp_path), subfolder="sub")
assert isinstance(pipeline, _DummyPipeline)
def test_subfolder_does_not_exist_raises(self, tmp_path):
"""Requesting a non-existent subfolder should raise (file not found)."""
write_config(tmp_path, self._minimal_config())
with pytest.raises(Exception):
Pipeline.from_pretrained(str(tmp_path), subfolder="missing")
def test_model_refs_resolved_under_subfolder(self, tmp_path):
"""$model/seg refs in config are expanded to <subfolder>/seg."""
config = {
"pipeline": {"name": _DUMMY_CLS_DOTPATH},
"params": {"segmentation": "$model/seg"},
}
write_config(tmp_path, config, subfolder="v1")
# Test expand_subfolders directly since loading would require real weights.
expand_subfolders(config, model_id=str(tmp_path), parent_subfolder="v1")
assert config["params"]["segmentation"]["subfolder"] == "v1/seg"
assert str(config["params"]["segmentation"]["checkpoint"]) == str(tmp_path)
def test_nested_subfolder_path(self, tmp_path):
"""subfolder can be a multi-level path like 'a/b'."""
write_config(tmp_path, self._minimal_config(), subfolder="a/b")
pipeline = Pipeline.from_pretrained(str(tmp_path), subfolder="a/b")
assert isinstance(pipeline, _DummyPipeline)
def test_model_refs_resolved_under_nested_subfolder(self, tmp_path):
"""$model/x under nested subfolder 'a/b' resolves to 'a/b/x'."""
config = {
"pipeline": {"name": _DUMMY_CLS_DOTPATH},
"params": {"embedding": "$model/emb"},
}
expand_subfolders(config, model_id="org/repo", parent_subfolder="a/b")
assert config["params"]["embedding"]["subfolder"] == "a/b/emb"
def test_revision_raises_with_local_directory(self, tmp_path):
"""Passing revision= with a local directory must raise ValueError."""
write_config(tmp_path, self._minimal_config())
with pytest.raises(ValueError, match="[Rr]evision"):
Pipeline.from_pretrained(str(tmp_path), revision="main")
# ---------------------------------------------------------------------------
# Nested pipeline tests: parent → sub → sub
# ---------------------------------------------------------------------------
class TestNestedSubfolderPipelines:
"""Parent pipeline loads a subpipeline which itself loads a subpipeline.
Directory layout used in these tests:
<root>/
v1/
config.yaml ← _DummyParentPipeline, sub: "$model/child"
child/
config.yaml ← _DummyParentPipeline, sub: "$model/grandchild"
grandchild/
config.yaml ← _DummyPipeline (leaf)
Loading with Pipeline.from_pretrained(<root>, subfolder="v1") must:
- expand "$model/child" → subfolder "v1/child"
- expand "$model/grandchild" → subfolder "v1/child/grandchild"
"""
def _write_three_level_tree(self, root: Path) -> None:
write_config(
root,
{"pipeline": {"name": _DUMMY_PARENT_CLS_DOTPATH, "params": {"sub": "$model/child"}}},
subfolder="v1",
)
write_config(
root,
{"pipeline": {"name": _DUMMY_PARENT_CLS_DOTPATH, "params": {"sub": "$model/grandchild"}}},
subfolder="v1/child",
)
write_config(
root,
{"pipeline": {"name": _DUMMY_CLS_DOTPATH}},
subfolder="v1/child/grandchild",
)
def test_parent_is_correct_type(self, tmp_path):
"""Top-level pipeline is a _DummyParentPipeline."""
self._write_three_level_tree(tmp_path)
parent = Pipeline.from_pretrained(str(tmp_path), subfolder="v1")
assert isinstance(parent, _DummyParentPipeline)
def test_subpipeline_is_registered(self, tmp_path):
"""Sub-pipeline is stored in parent._pipelines under the 'sub' key."""
self._write_three_level_tree(tmp_path)
parent = Pipeline.from_pretrained(str(tmp_path), subfolder="v1")
assert "sub" in parent._pipelines
def test_subpipeline_is_correct_type(self, tmp_path):
"""Sub-pipeline is itself a _DummyParentPipeline."""
self._write_three_level_tree(tmp_path)
parent = Pipeline.from_pretrained(str(tmp_path), subfolder="v1")
assert isinstance(parent.sub, _DummyParentPipeline)
def test_grandchild_pipeline_is_registered(self, tmp_path):
"""Grand-child pipeline is stored in child._pipelines under 'sub'."""
self._write_three_level_tree(tmp_path)
parent = Pipeline.from_pretrained(str(tmp_path), subfolder="v1")
assert "sub" in parent.sub._pipelines
def test_grandchild_pipeline_is_leaf_type(self, tmp_path):
"""Grand-child pipeline is a _DummyPipeline (the leaf class)."""
self._write_three_level_tree(tmp_path)
parent = Pipeline.from_pretrained(str(tmp_path), subfolder="v1")
assert isinstance(parent.sub.sub, _DummyPipeline)
def test_without_subfolder_loads_from_root(self, tmp_path):
"""Without subfolder=, the root config.yaml is used (one-level tree)."""
write_config(
tmp_path,
{"pipeline": {"name": _DUMMY_PARENT_CLS_DOTPATH, "params": {"sub": "$model/child"}}},
)
write_config(tmp_path, {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}, subfolder="child")
parent = Pipeline.from_pretrained(str(tmp_path))
assert isinstance(parent, _DummyParentPipeline)
assert isinstance(parent.sub, _DummyPipeline)
# ---------------------------------------------------------------------------
# Tests for dict and explicit-file-path checkpoints with subfolder
# ---------------------------------------------------------------------------
class TestFromPretrainedNonSubfolderCases:
"""subfolder must NOT be forwarded as parent_subfolder when the config
was given as a dict or an explicit file path — in those cases subfolder
played no role in locating config.yaml, so prepending it would wrongly
rewrite $model/... references."""
def test_dict_checkpoint_with_subfolder_does_not_double_prefix(self, tmp_path):
"""When checkpoint is a dict, $model refs must not gain the subfolder prefix."""
config = {
"pipeline": {"name": _DUMMY_CLS_DOTPATH},
"params": {"seg": "$model/seg"},
}
# Pass subfolder='v1' — it must be ignored for expand_subfolders.
expand_subfolders(config, model_id=str(tmp_path), parent_subfolder=None)
assert config["params"]["seg"]["subfolder"] == "seg"
def test_file_checkpoint_with_subfolder_raises(self, tmp_path):
"""Passing subfolder= together with an explicit file path must raise ValueError."""
config = {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}
cfg_path = write_config(tmp_path, config)
with pytest.raises(ValueError, match="[Ss]ubfolder"):
Pipeline.from_pretrained(str(cfg_path), subfolder="v1")
def test_file_checkpoint_without_subfolder_loads(self, tmp_path):
"""Explicit file path without subfolder= loads the pipeline correctly."""
config = {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}
cfg_path = write_config(tmp_path, config)
pipeline = Pipeline.from_pretrained(str(cfg_path))
assert isinstance(pipeline, _DummyPipeline)
def test_dict_checkpoint_with_subfolder_raises(self):
"""Passing subfolder= together with a dict checkpoint must raise ValueError."""
config = {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}
with pytest.raises(ValueError, match="[Ss]ubfolder"):
Pipeline.from_pretrained(config, subfolder="v1")
def test_dict_checkpoint_end_to_end(self, tmp_path, monkeypatch):
"""Pipeline.from_pretrained(dict) resolves $model refs relative to cwd."""
# The dict branch uses Path.cwd() as model_id, so chdir to tmp_path.
monkeypatch.chdir(tmp_path)
child_config = {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}
write_config(tmp_path, child_config, subfolder="seg")
config = {
"pipeline": {"name": _DUMMY_PARENT_CLS_DOTPATH, "params": {"sub": "$model/seg"}},
}
pipeline = Pipeline.from_pretrained(config)
assert isinstance(pipeline, _DummyParentPipeline)