Files
pyannote-audio/tests/test_pipeline_subfolder.py
T
ea18bc70d8 Align audio CI and releases; require Python 3.11 (#2081)
* feat(cli): add --subfolder option to benchmark command (#2063)
* fix(utils): fix `SpeakerDiarizationMixin.optimal_mapping` when `reference` is a mapping
* fix(task): fix `balance` option crashing training with `KeyError: 'metadata'` (#2067)
* fix(io): fix `Audio.crop` raising when an in-memory waveform is cropped up to its very end (#2068)
* fix(model): make Model.default_activation return identity for unsupported tasks
* fix: compute sample duration from preloaded waveform
* fix(pipeline): omit unset runtime options from model references (#2080)

Co-authored-by: abeeranajam31 <128603901+abeeranajam31@users.noreply.github.com>
Co-authored-by: antoinelaurent <antoinelaurent@me.com>
Co-authored-by: Kayvan Zahiri <123409429+Kayvan-Zahiri@users.noreply.github.com>
Co-authored-by: Mohammad Hijjawi <51020382+MohammadHijjawi97@users.noreply.github.com>
2026-10-22 15:07:01 +02:00

404 lines
17 KiB
Python

# MIT License
#
# Copyright (c) 2025- pyannoteAI
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""Tests for subfolder support in Pipeline.from_pretrained"""
from pathlib import Path
import pytest
import yaml
from pyannote.audio.core.pipeline import Pipeline, expand_subfolders
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def write_config(directory: Path, config: dict, subfolder: str | None = None) -> Path:
"""Write a config.yaml inside *directory* (or *directory/subfolder*)."""
target = directory / subfolder if subfolder else directory
target.mkdir(parents=True, exist_ok=True)
cfg_path = target / "config.yaml"
cfg_path.write_text(yaml.dump(config))
return cfg_path
# from_pretrained always injects `token` and `cache_dir` into params via
# setdefault() before calling Klass(**params), so the subclass __init__ must
# accept (and ignore) them.
class _DummyPipeline(Pipeline):
"""Minimal concrete subclass of Pipeline for testing from_pretrained."""
def __init__(self, token=None, cache_dir=None, **kwargs):
super().__init__()
def apply(self, file, **kwargs):
return None
_DUMMY_CLS_DOTPATH = f"{_DummyPipeline.__module__}.{_DummyPipeline.__qualname__}"
class _DummyParentPipeline(Pipeline):
"""Pipeline that owns a single subpipeline loaded from a checkpoint dict."""
def __init__(self, sub=None, token=None, cache_dir=None):
super().__init__()
if isinstance(sub, dict):
# Assigning a Pipeline instance triggers __setattr__ which registers
# it in self._pipelines — the standard pyannote sub-pipeline pattern.
self.sub = Pipeline.from_pretrained(**sub)
def apply(self, file, **kwargs):
return None
_DUMMY_PARENT_CLS_DOTPATH = (
f"{_DummyParentPipeline.__module__}.{_DummyParentPipeline.__qualname__}"
)
# ---------------------------------------------------------------------------
# expand_subfolders unit tests
# ---------------------------------------------------------------------------
class TestExpandSubfolders:
"""expand_subfolders resolves $model/... references in config dicts/lists."""
def test_no_model_references(self):
config = {"key": "plain_value", "nested": {"k": 42}}
expand_subfolders(config, model_id="org/repo")
assert config == {"key": "plain_value", "nested": {"k": 42}}
def test_dict_model_reference_no_parent_subfolder(self):
config = {"embedding": "$model/embeddings"}
expand_subfolders(config, model_id="org/repo", token="tok")
assert config["embedding"] == {
"checkpoint": "org/repo",
"revision": None,
"subfolder": "embeddings",
"token": "tok",
}
@pytest.mark.parametrize("as_list", [False, True])
@pytest.mark.parametrize("token", [None, False, "tok"])
@pytest.mark.parametrize("cache_dir", [None, Path("cache")])
def test_optional_runtime_options(self, as_list, token, cache_dir):
config = ["$model/weights"] if as_list else {"model": "$model/weights"}
expand_subfolders(config, model_id="org/repo", token=token, cache_dir=cache_dir)
model = config[0] if as_list else config["model"]
for key, value in {"token": token, "cache_dir": cache_dir}.items():
if value is None:
assert key not in model
else:
assert model[key] == value
def test_dict_model_reference_with_parent_subfolder(self):
"""Child $model refs are resolved under parent_subfolder."""
config = {"segmentation": "$model/seg"}
expand_subfolders(
config,
model_id="org/repo",
parent_subfolder="v1",
token=None,
)
assert config["segmentation"]["subfolder"] == "v1/seg"
def test_dict_model_reference_with_explicit_revision(self):
"""@revision in $model/path@rev overrides parent_revision."""
config = {"model": "$model/weights@abc123"}
expand_subfolders(config, model_id="org/repo", parent_revision="main")
assert config["model"]["revision"] == "abc123"
assert config["model"]["subfolder"] == "weights"
def test_dict_model_reference_inherits_parent_revision(self):
config = {"model": "$model/weights"}
expand_subfolders(config, model_id="org/repo", parent_revision="v2")
assert config["model"]["revision"] == "v2"
def test_list_model_reference_no_parent_subfolder(self):
config = ["$model/part_a", "$model/part_b"]
expand_subfolders(config, model_id="org/repo")
assert config[0]["subfolder"] == "part_a"
assert config[1]["subfolder"] == "part_b"
def test_list_model_reference_with_parent_subfolder(self):
config = ["$model/seg", "plain"]
expand_subfolders(config, model_id="org/repo", parent_subfolder="root")
assert config[0]["subfolder"] == "root/seg"
assert config[1] == "plain"
def test_nested_dict_model_references(self):
"""Recursion into nested dicts."""
config = {"outer": {"inner": "$model/deep"}}
expand_subfolders(config, model_id="org/repo", parent_subfolder="base")
assert config["outer"]["inner"]["subfolder"] == "base/deep"
def test_nested_list_model_references(self):
"""Recursion into lists that contain dicts."""
config = {"items": [{"model": "$model/a"}, {"model": "$model/b"}]}
expand_subfolders(config, model_id="org/repo", parent_subfolder="p")
assert config["items"][0]["model"]["subfolder"] == "p/a"
assert config["items"][1]["model"]["subfolder"] == "p/b"
def test_no_model_id_leaves_checkpoint_as_none(self):
"""When model_id is None the checkpoint field is also None."""
config = {"seg": "$model/seg"}
expand_subfolders(config, model_id=None)
assert config["seg"]["checkpoint"] is None
def test_parent_subfolder_combined_with_explicit_revision(self):
"""parent_subfolder and explicit @revision can coexist."""
config = {"model": "$model/weights@pinned"}
expand_subfolders(
config,
model_id="org/repo",
parent_subfolder="folder",
parent_revision="ignored",
)
assert config["model"]["subfolder"] == "folder/weights"
assert config["model"]["revision"] == "pinned"
# ---------------------------------------------------------------------------
# Pipeline.from_pretrained local-directory tests
# ---------------------------------------------------------------------------
class TestFromPretrainedLocalSubfolder:
"""Pipeline.from_pretrained with local directories and the subfolder arg."""
def _minimal_config(self) -> dict:
return {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}
def test_loads_config_from_root_without_subfolder(self, tmp_path):
"""Without subfolder, config.yaml is read from the directory root."""
write_config(tmp_path, self._minimal_config())
pipeline = Pipeline.from_pretrained(str(tmp_path))
assert isinstance(pipeline, _DummyPipeline)
def test_loads_config_from_subfolder(self, tmp_path):
"""With subfolder='sub', config.yaml is read from <root>/sub/config.yaml."""
write_config(tmp_path, self._minimal_config(), subfolder="sub")
pipeline = Pipeline.from_pretrained(str(tmp_path), subfolder="sub")
assert isinstance(pipeline, _DummyPipeline)
def test_subfolder_does_not_exist_raises(self, tmp_path):
"""Requesting a non-existent subfolder should raise (file not found)."""
write_config(tmp_path, self._minimal_config())
with pytest.raises(FileNotFoundError):
Pipeline.from_pretrained(str(tmp_path), subfolder="missing")
def test_model_refs_resolved_under_subfolder(self, tmp_path):
"""$model/seg refs in config are expanded to <subfolder>/seg."""
config = {
"pipeline": {"name": _DUMMY_CLS_DOTPATH},
"params": {"segmentation": "$model/seg"},
}
write_config(tmp_path, config, subfolder="v1")
# Test expand_subfolders directly since loading would require real weights.
expand_subfolders(config, model_id=str(tmp_path), parent_subfolder="v1")
assert config["params"]["segmentation"]["subfolder"] == "v1/seg"
assert str(config["params"]["segmentation"]["checkpoint"]) == str(tmp_path)
def test_nested_subfolder_path(self, tmp_path):
"""subfolder can be a multi-level path like 'a/b'."""
write_config(tmp_path, self._minimal_config(), subfolder="a/b")
pipeline = Pipeline.from_pretrained(str(tmp_path), subfolder="a/b")
assert isinstance(pipeline, _DummyPipeline)
def test_model_refs_resolved_under_nested_subfolder(self, tmp_path):
"""$model/x under nested subfolder 'a/b' resolves to 'a/b/x'."""
config = {
"pipeline": {"name": _DUMMY_CLS_DOTPATH},
"params": {"embedding": "$model/emb"},
}
expand_subfolders(config, model_id="org/repo", parent_subfolder="a/b")
assert config["params"]["embedding"]["subfolder"] == "a/b/emb"
def test_revision_raises_with_local_directory(self, tmp_path):
"""Passing revision= with a local directory must raise ValueError."""
write_config(tmp_path, self._minimal_config())
with pytest.raises(ValueError, match="[Rr]evision"):
Pipeline.from_pretrained(str(tmp_path), revision="main")
# ---------------------------------------------------------------------------
# Nested pipeline tests: parent → sub → sub
# ---------------------------------------------------------------------------
class TestNestedSubfolderPipelines:
"""Parent pipeline loads a subpipeline which itself loads a subpipeline.
Directory layout used in these tests:
<root>/
v1/
config.yaml ← _DummyParentPipeline, sub: "$model/child"
child/
config.yaml ← _DummyParentPipeline, sub: "$model/grandchild"
grandchild/
config.yaml ← _DummyPipeline (leaf)
Loading with Pipeline.from_pretrained(<root>, subfolder="v1") must:
- expand "$model/child" → subfolder "v1/child"
- expand "$model/grandchild" → subfolder "v1/child/grandchild"
"""
def _write_three_level_tree(self, root: Path) -> None:
write_config(
root,
{
"pipeline": {
"name": _DUMMY_PARENT_CLS_DOTPATH,
"params": {"sub": "$model/child"},
}
},
subfolder="v1",
)
write_config(
root,
{
"pipeline": {
"name": _DUMMY_PARENT_CLS_DOTPATH,
"params": {"sub": "$model/grandchild"},
}
},
subfolder="v1/child",
)
write_config(
root,
{"pipeline": {"name": _DUMMY_CLS_DOTPATH}},
subfolder="v1/child/grandchild",
)
def test_parent_is_correct_type(self, tmp_path):
"""Top-level pipeline is a _DummyParentPipeline."""
self._write_three_level_tree(tmp_path)
parent = Pipeline.from_pretrained(str(tmp_path), subfolder="v1")
assert isinstance(parent, _DummyParentPipeline)
def test_subpipeline_is_registered(self, tmp_path):
"""Sub-pipeline is stored in parent._pipelines under the 'sub' key."""
self._write_three_level_tree(tmp_path)
parent = Pipeline.from_pretrained(str(tmp_path), subfolder="v1")
assert "sub" in parent._pipelines
def test_subpipeline_is_correct_type(self, tmp_path):
"""Sub-pipeline is itself a _DummyParentPipeline."""
self._write_three_level_tree(tmp_path)
parent = Pipeline.from_pretrained(str(tmp_path), subfolder="v1")
assert isinstance(parent.sub, _DummyParentPipeline)
def test_grandchild_pipeline_is_registered(self, tmp_path):
"""Grand-child pipeline is stored in child._pipelines under 'sub'."""
self._write_three_level_tree(tmp_path)
parent = Pipeline.from_pretrained(str(tmp_path), subfolder="v1")
assert "sub" in parent.sub._pipelines
def test_grandchild_pipeline_is_leaf_type(self, tmp_path):
"""Grand-child pipeline is a _DummyPipeline (the leaf class)."""
self._write_three_level_tree(tmp_path)
parent = Pipeline.from_pretrained(str(tmp_path), subfolder="v1")
assert isinstance(parent.sub.sub, _DummyPipeline)
def test_without_subfolder_loads_from_root(self, tmp_path):
"""Without subfolder=, the root config.yaml is used (one-level tree)."""
write_config(
tmp_path,
{
"pipeline": {
"name": _DUMMY_PARENT_CLS_DOTPATH,
"params": {"sub": "$model/child"},
}
},
)
write_config(
tmp_path, {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}, subfolder="child"
)
parent = Pipeline.from_pretrained(str(tmp_path))
assert isinstance(parent, _DummyParentPipeline)
assert isinstance(parent.sub, _DummyPipeline)
# ---------------------------------------------------------------------------
# Tests for dict and explicit-file-path checkpoints with subfolder
# ---------------------------------------------------------------------------
class TestFromPretrainedNonSubfolderCases:
"""subfolder must NOT be forwarded as parent_subfolder when the config
was given as a dict or an explicit file path — in those cases subfolder
played no role in locating config.yaml, so prepending it would wrongly
rewrite $model/... references."""
def test_dict_checkpoint_with_subfolder_does_not_double_prefix(self, tmp_path):
"""When checkpoint is a dict, $model refs must not gain the subfolder prefix."""
config = {
"pipeline": {"name": _DUMMY_CLS_DOTPATH},
"params": {"seg": "$model/seg"},
}
# Pass subfolder='v1' — it must be ignored for expand_subfolders.
expand_subfolders(config, model_id=str(tmp_path), parent_subfolder=None)
assert config["params"]["seg"]["subfolder"] == "seg"
def test_file_checkpoint_with_subfolder_raises(self, tmp_path):
"""Passing subfolder= together with an explicit file path must raise ValueError."""
config = {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}
cfg_path = write_config(tmp_path, config)
with pytest.raises(ValueError, match="[Ss]ubfolder"):
Pipeline.from_pretrained(str(cfg_path), subfolder="v1")
def test_file_checkpoint_without_subfolder_loads(self, tmp_path):
"""Explicit file path without subfolder= loads the pipeline correctly."""
config = {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}
cfg_path = write_config(tmp_path, config)
pipeline = Pipeline.from_pretrained(str(cfg_path))
assert isinstance(pipeline, _DummyPipeline)
def test_dict_checkpoint_with_subfolder_raises(self):
"""Passing subfolder= together with a dict checkpoint must raise ValueError."""
config = {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}
with pytest.raises(ValueError, match="[Ss]ubfolder"):
Pipeline.from_pretrained(config, subfolder="v1")
def test_dict_checkpoint_end_to_end(self, tmp_path, monkeypatch):
"""Pipeline.from_pretrained(dict) resolves $model refs relative to cwd."""
# The dict branch uses Path.cwd() as model_id, so chdir to tmp_path.
monkeypatch.chdir(tmp_path)
child_config = {"pipeline": {"name": _DUMMY_CLS_DOTPATH}}
write_config(tmp_path, child_config, subfolder="seg")
config = {
"pipeline": {
"name": _DUMMY_PARENT_CLS_DOTPATH,
"params": {"sub": "$model/seg"},
},
}
pipeline = Pipeline.from_pretrained(config)
assert isinstance(pipeline, _DummyParentPipeline)