mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
318 lines
10 KiB
Python
318 lines
10 KiB
Python
# MIT License
|
|
#
|
|
# Copyright (c) 2026- pyannoteAI
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in all
|
|
# copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import yaml
|
|
from pyannote.core import Annotation, Segment
|
|
from pyannote.pipeline import Pipeline
|
|
from pyannote.pipeline.optimizer import Optimizer
|
|
from pyannote.pipeline.parameter import Integer
|
|
from typer.testing import CliRunner
|
|
|
|
from pyannote.audio.__main__ import Metric, app
|
|
from pyannote.audio.__main__ import Pipeline as AudioPipeline
|
|
|
|
|
|
class MockPipeline(Pipeline):
|
|
"""Minimal pipeline exercising the real CLI/optimizer integration."""
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.threshold = Integer(0, 1)
|
|
|
|
def __call__(self, current_file, **kwargs):
|
|
return SimpleNamespace(speaker_diarization=current_file["annotation"])
|
|
|
|
def default_parameters(self):
|
|
return {"threshold": 0}
|
|
|
|
def to(self, device):
|
|
return self
|
|
|
|
|
|
class MockProtocol:
|
|
def development(self):
|
|
annotation = Annotation(uri="file")
|
|
annotation[Segment(0.0, 1.0)] = "speaker"
|
|
yield {
|
|
"uri": "file",
|
|
"annotation": annotation,
|
|
"annotated": annotation.get_timeline(),
|
|
}
|
|
|
|
|
|
def test_optimize_cli_with_single_objective(tmp_path, monkeypatch):
|
|
"""Scalar CLI optimization remains compatible with the new Optimizer API."""
|
|
pipeline_yml = tmp_path / "pipeline.yaml"
|
|
pipeline_yml.write_text("pipeline: mock\n")
|
|
|
|
mock_pipeline = MockPipeline()
|
|
monkeypatch.setattr(
|
|
AudioPipeline,
|
|
"from_pretrained",
|
|
lambda *args, **kwargs: mock_pipeline,
|
|
)
|
|
monkeypatch.setattr(
|
|
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
|
|
lambda *args, **kwargs: MockProtocol(),
|
|
)
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"optimize",
|
|
str(pipeline_yml),
|
|
"Debug.SpeakerDiarization.Debug",
|
|
"--subset",
|
|
"development",
|
|
"--device",
|
|
"cpu",
|
|
"--max-iterations",
|
|
"1",
|
|
"--average-case",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0, result.output
|
|
|
|
optimized_yml = tmp_path / (
|
|
"pipeline.Debug.SpeakerDiarization.Debug.development.DiarizationErrorRate.yaml"
|
|
)
|
|
optimized = yaml.safe_load(optimized_yml.read_text())
|
|
assert optimized["params"] == {"threshold": 0}
|
|
assert optimized["optimization"]["status"]["best_loss"] == 0.0
|
|
|
|
|
|
def test_optimize_cli_with_multiple_objectives(tmp_path, monkeypatch):
|
|
"""Repeated metric options produce a named Pareto-front manifest."""
|
|
pipeline_yml = tmp_path / "pipeline.yaml"
|
|
pipeline_yml.write_text("pipeline: mock\n")
|
|
|
|
mock_pipeline = MockPipeline()
|
|
monkeypatch.setattr(
|
|
AudioPipeline,
|
|
"from_pretrained",
|
|
lambda *args, **kwargs: mock_pipeline,
|
|
)
|
|
monkeypatch.setattr(
|
|
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
|
|
lambda *args, **kwargs: MockProtocol(),
|
|
)
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"optimize",
|
|
str(pipeline_yml),
|
|
"Debug.SpeakerDiarization.Debug",
|
|
"--subset",
|
|
"development",
|
|
"--device",
|
|
"cpu",
|
|
"--max-iterations",
|
|
"1",
|
|
"--metric",
|
|
"DiarizationPurity",
|
|
"--metric",
|
|
"DiarizationCoverage",
|
|
"--average-case",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0, result.output
|
|
|
|
pareto_yml = tmp_path / (
|
|
"pipeline.Debug.SpeakerDiarization.Debug.development."
|
|
"DiarizationPurity+DiarizationCoverage.pareto.yaml"
|
|
)
|
|
pareto = yaml.safe_load(pareto_yml.read_text())["optimization"]
|
|
assert pareto["metrics"] == ["DiarizationPurity", "DiarizationCoverage"]
|
|
assert pareto["status"]["pareto_front"] == [
|
|
{
|
|
"trial": 0,
|
|
"values": {
|
|
"DiarizationPurity": 1.0,
|
|
"DiarizationCoverage": 1.0,
|
|
},
|
|
"params": {"threshold": 0},
|
|
}
|
|
]
|
|
|
|
|
|
def test_multi_objective_export_refreshes_pareto_front(tmp_path, monkeypatch):
|
|
pipeline_yml = tmp_path / "pipeline.yaml"
|
|
pipeline_yml.write_text("pipeline: mock\n")
|
|
|
|
monkeypatch.setattr(
|
|
AudioPipeline, "from_pretrained", lambda *args, **kwargs: MockPipeline()
|
|
)
|
|
monkeypatch.setattr(
|
|
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
|
|
lambda *args, **kwargs: MockProtocol(),
|
|
)
|
|
monkeypatch.setattr(
|
|
Optimizer,
|
|
"tune_iter",
|
|
lambda self, *args, **kwargs: iter(
|
|
[
|
|
{
|
|
"pareto_front": [
|
|
{
|
|
"number": 0,
|
|
"values": (0.5, 0.5),
|
|
"params": {"threshold": 0},
|
|
}
|
|
]
|
|
}
|
|
]
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
Optimizer,
|
|
"pareto_front",
|
|
property(
|
|
lambda self: [
|
|
{
|
|
"number": 1,
|
|
"values": (0.25, 0.75),
|
|
"params": {"threshold": 1},
|
|
}
|
|
]
|
|
),
|
|
)
|
|
|
|
result = CliRunner().invoke(
|
|
app,
|
|
[
|
|
"optimize",
|
|
str(pipeline_yml),
|
|
"Debug.SpeakerDiarization.Debug",
|
|
"--subset",
|
|
"development",
|
|
"--device",
|
|
"cpu",
|
|
"--max-iterations",
|
|
"1",
|
|
"--metric",
|
|
"DiarizationPurity",
|
|
"--metric",
|
|
"DiarizationCoverage",
|
|
"--average-case",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0, result.output
|
|
output = tmp_path / (
|
|
"pipeline.Debug.SpeakerDiarization.Debug.development."
|
|
"DiarizationPurity+DiarizationCoverage.pareto.yaml"
|
|
)
|
|
pareto = yaml.safe_load(output.read_text())["optimization"]["status"][
|
|
"pareto_front"
|
|
]
|
|
assert pareto == [
|
|
{
|
|
"trial": 1,
|
|
"values": {"DiarizationPurity": 0.25, "DiarizationCoverage": 0.75},
|
|
"params": {"threshold": 1},
|
|
}
|
|
]
|
|
assert (tmp_path / f"{output.name}.lock").exists()
|
|
assert not list(tmp_path.glob(f".{output.name}.*.tmp"))
|
|
|
|
|
|
@pytest.mark.parametrize("multi_objective", [False, True])
|
|
def test_optimize_cli_with_speaker_count_error(tmp_path, monkeypatch, multi_objective):
|
|
"""Count errors respect UEM and average files equally in both optimization modes."""
|
|
|
|
class SpeakerCountPipeline(MockPipeline):
|
|
def __call__(self, current_file, **kwargs):
|
|
return SimpleNamespace(speaker_diarization=current_file["hypothesis"])
|
|
|
|
class SpeakerCountProtocol:
|
|
def development(self):
|
|
for index, (duration, extra_speakers) in enumerate([(1.0, 1), (10.0, 3)]):
|
|
reference = Annotation(uri=f"file{index}")
|
|
reference[Segment(0.0, duration)] = "speaker"
|
|
hypothesis = reference.copy()
|
|
for speaker in range(extra_speakers):
|
|
hypothesis[Segment(0.0, duration), speaker + 1] = f"extra{speaker}"
|
|
hypothesis[Segment(duration, duration + 1.0)] = "outside_uem"
|
|
yield {
|
|
"uri": reference.uri,
|
|
"annotation": reference,
|
|
"annotated": reference.get_timeline(),
|
|
"hypothesis": hypothesis,
|
|
}
|
|
|
|
pipeline_yml = tmp_path / "pipeline.yaml"
|
|
pipeline_yml.write_text("pipeline: mock\n")
|
|
mock_pipeline = SpeakerCountPipeline()
|
|
monkeypatch.setattr(
|
|
AudioPipeline, "from_pretrained", lambda *args, **kwargs: mock_pipeline
|
|
)
|
|
monkeypatch.setattr(
|
|
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
|
|
lambda *args, **kwargs: SpeakerCountProtocol(),
|
|
)
|
|
|
|
metrics = ["DiarizationSpeakerCountError"]
|
|
if multi_objective:
|
|
metrics.append("DiarizationCoverage")
|
|
metric_options = [option for metric in metrics for option in ("--metric", metric)]
|
|
result = CliRunner().invoke(
|
|
app,
|
|
[
|
|
"optimize",
|
|
str(pipeline_yml),
|
|
"Debug.SpeakerDiarization.Debug",
|
|
"--device",
|
|
"cpu",
|
|
"--max-iterations",
|
|
"1",
|
|
"--average-case",
|
|
*metric_options,
|
|
],
|
|
)
|
|
assert result.exit_code == 0, result.output
|
|
assert Metric.DiarizationSpeakerCountError.direction == "minimize"
|
|
|
|
suffix = ".pareto.yaml" if multi_objective else ".yaml"
|
|
output = tmp_path / (
|
|
"pipeline.Debug.SpeakerDiarization.Debug.development."
|
|
+ "+".join(metrics)
|
|
+ suffix
|
|
)
|
|
optimization = yaml.safe_load(output.read_text())["optimization"]
|
|
if multi_objective:
|
|
assert mock_pipeline.get_direction() == ("minimize", "maximize")
|
|
assert optimization["metrics"] == metrics
|
|
assert optimization["status"]["pareto_front"][0]["values"] == {
|
|
"DiarizationSpeakerCountError": 2.0,
|
|
"DiarizationCoverage": 1.0,
|
|
}
|
|
else:
|
|
assert optimization["status"]["best_loss"] == 2.0
|