Files
pyannote-audio/tests/test_cli_optimize.py

318 lines
10 KiB
Python

# MIT License
#
# Copyright (c) 2026- pyannoteAI
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from types import SimpleNamespace
import pytest
import yaml
from pyannote.core import Annotation, Segment
from pyannote.pipeline import Pipeline
from pyannote.pipeline.optimizer import Optimizer
from pyannote.pipeline.parameter import Integer
from typer.testing import CliRunner
from pyannote.audio.__main__ import Metric, app
from pyannote.audio.__main__ import Pipeline as AudioPipeline
class MockPipeline(Pipeline):
"""Minimal pipeline exercising the real CLI/optimizer integration."""
def __init__(self):
super().__init__()
self.threshold = Integer(0, 1)
def __call__(self, current_file, **kwargs):
return SimpleNamespace(speaker_diarization=current_file["annotation"])
def default_parameters(self):
return {"threshold": 0}
def to(self, device):
return self
class MockProtocol:
def development(self):
annotation = Annotation(uri="file")
annotation[Segment(0.0, 1.0)] = "speaker"
yield {
"uri": "file",
"annotation": annotation,
"annotated": annotation.get_timeline(),
}
def test_optimize_cli_with_single_objective(tmp_path, monkeypatch):
"""Scalar CLI optimization remains compatible with the new Optimizer API."""
pipeline_yml = tmp_path / "pipeline.yaml"
pipeline_yml.write_text("pipeline: mock\n")
mock_pipeline = MockPipeline()
monkeypatch.setattr(
AudioPipeline,
"from_pretrained",
lambda *args, **kwargs: mock_pipeline,
)
monkeypatch.setattr(
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
lambda *args, **kwargs: MockProtocol(),
)
runner = CliRunner()
result = runner.invoke(
app,
[
"optimize",
str(pipeline_yml),
"Debug.SpeakerDiarization.Debug",
"--subset",
"development",
"--device",
"cpu",
"--max-iterations",
"1",
"--average-case",
],
)
assert result.exit_code == 0, result.output
optimized_yml = tmp_path / (
"pipeline.Debug.SpeakerDiarization.Debug.development.DiarizationErrorRate.yaml"
)
optimized = yaml.safe_load(optimized_yml.read_text())
assert optimized["params"] == {"threshold": 0}
assert optimized["optimization"]["status"]["best_loss"] == 0.0
def test_optimize_cli_with_multiple_objectives(tmp_path, monkeypatch):
"""Repeated metric options produce a named Pareto-front manifest."""
pipeline_yml = tmp_path / "pipeline.yaml"
pipeline_yml.write_text("pipeline: mock\n")
mock_pipeline = MockPipeline()
monkeypatch.setattr(
AudioPipeline,
"from_pretrained",
lambda *args, **kwargs: mock_pipeline,
)
monkeypatch.setattr(
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
lambda *args, **kwargs: MockProtocol(),
)
runner = CliRunner()
result = runner.invoke(
app,
[
"optimize",
str(pipeline_yml),
"Debug.SpeakerDiarization.Debug",
"--subset",
"development",
"--device",
"cpu",
"--max-iterations",
"1",
"--metric",
"DiarizationPurity",
"--metric",
"DiarizationCoverage",
"--average-case",
],
)
assert result.exit_code == 0, result.output
pareto_yml = tmp_path / (
"pipeline.Debug.SpeakerDiarization.Debug.development."
"DiarizationPurity+DiarizationCoverage.pareto.yaml"
)
pareto = yaml.safe_load(pareto_yml.read_text())["optimization"]
assert pareto["metrics"] == ["DiarizationPurity", "DiarizationCoverage"]
assert pareto["status"]["pareto_front"] == [
{
"trial": 0,
"values": {
"DiarizationPurity": 1.0,
"DiarizationCoverage": 1.0,
},
"params": {"threshold": 0},
}
]
def test_multi_objective_export_refreshes_pareto_front(tmp_path, monkeypatch):
pipeline_yml = tmp_path / "pipeline.yaml"
pipeline_yml.write_text("pipeline: mock\n")
monkeypatch.setattr(
AudioPipeline, "from_pretrained", lambda *args, **kwargs: MockPipeline()
)
monkeypatch.setattr(
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
lambda *args, **kwargs: MockProtocol(),
)
monkeypatch.setattr(
Optimizer,
"tune_iter",
lambda self, *args, **kwargs: iter(
[
{
"pareto_front": [
{
"number": 0,
"values": (0.5, 0.5),
"params": {"threshold": 0},
}
]
}
]
),
)
monkeypatch.setattr(
Optimizer,
"pareto_front",
property(
lambda self: [
{
"number": 1,
"values": (0.25, 0.75),
"params": {"threshold": 1},
}
]
),
)
result = CliRunner().invoke(
app,
[
"optimize",
str(pipeline_yml),
"Debug.SpeakerDiarization.Debug",
"--subset",
"development",
"--device",
"cpu",
"--max-iterations",
"1",
"--metric",
"DiarizationPurity",
"--metric",
"DiarizationCoverage",
"--average-case",
],
)
assert result.exit_code == 0, result.output
output = tmp_path / (
"pipeline.Debug.SpeakerDiarization.Debug.development."
"DiarizationPurity+DiarizationCoverage.pareto.yaml"
)
pareto = yaml.safe_load(output.read_text())["optimization"]["status"][
"pareto_front"
]
assert pareto == [
{
"trial": 1,
"values": {"DiarizationPurity": 0.25, "DiarizationCoverage": 0.75},
"params": {"threshold": 1},
}
]
assert (tmp_path / f"{output.name}.lock").exists()
assert not list(tmp_path.glob(f".{output.name}.*.tmp"))
@pytest.mark.parametrize("multi_objective", [False, True])
def test_optimize_cli_with_speaker_count_error(tmp_path, monkeypatch, multi_objective):
"""Count errors respect UEM and average files equally in both optimization modes."""
class SpeakerCountPipeline(MockPipeline):
def __call__(self, current_file, **kwargs):
return SimpleNamespace(speaker_diarization=current_file["hypothesis"])
class SpeakerCountProtocol:
def development(self):
for index, (duration, extra_speakers) in enumerate([(1.0, 1), (10.0, 3)]):
reference = Annotation(uri=f"file{index}")
reference[Segment(0.0, duration)] = "speaker"
hypothesis = reference.copy()
for speaker in range(extra_speakers):
hypothesis[Segment(0.0, duration), speaker + 1] = f"extra{speaker}"
hypothesis[Segment(duration, duration + 1.0)] = "outside_uem"
yield {
"uri": reference.uri,
"annotation": reference,
"annotated": reference.get_timeline(),
"hypothesis": hypothesis,
}
pipeline_yml = tmp_path / "pipeline.yaml"
pipeline_yml.write_text("pipeline: mock\n")
mock_pipeline = SpeakerCountPipeline()
monkeypatch.setattr(
AudioPipeline, "from_pretrained", lambda *args, **kwargs: mock_pipeline
)
monkeypatch.setattr(
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
lambda *args, **kwargs: SpeakerCountProtocol(),
)
metrics = ["DiarizationSpeakerCountError"]
if multi_objective:
metrics.append("DiarizationCoverage")
metric_options = [option for metric in metrics for option in ("--metric", metric)]
result = CliRunner().invoke(
app,
[
"optimize",
str(pipeline_yml),
"Debug.SpeakerDiarization.Debug",
"--device",
"cpu",
"--max-iterations",
"1",
"--average-case",
*metric_options,
],
)
assert result.exit_code == 0, result.output
assert Metric.DiarizationSpeakerCountError.direction == "minimize"
suffix = ".pareto.yaml" if multi_objective else ".yaml"
output = tmp_path / (
"pipeline.Debug.SpeakerDiarization.Debug.development."
+ "+".join(metrics)
+ suffix
)
optimization = yaml.safe_load(output.read_text())["optimization"]
if multi_objective:
assert mock_pipeline.get_direction() == ("minimize", "maximize")
assert optimization["metrics"] == metrics
assert optimization["status"]["pareto_front"][0]["values"] == {
"DiarizationSpeakerCountError": 2.0,
"DiarizationCoverage": 1.0,
}
else:
assert optimization["status"]["best_loss"] == 2.0