feat(cli): add support for multi-objective optimzation (#2056)

This commit is contained in:
Hervé BREDIN authored and GitHub committed 2026-11-06 17:14:43 +01:00
1 parent cc679dc7d6
commit 08d6f3160a
7 files changed
+1024 -30

No files matched your search

+1 -1
View File
@@ -7,10 +7,10 @@
- fix(model): restore `fbank_only` mode in WeSpeaker embedding models
- fix(task): avoid training crashes with `spawn`/`forkserver` workers, including
Python 3.14's default on Linux, by warning and using `num_workers=0`
- fix(pipeline): omit unset `token` and `cache_dir` when expanding `$model/` references so loaders can inherit runtime options
- fix(model): honor the configured linear hidden size in `ToTaToNet` [@antonsoo](https://github.com/antonsoo)
- fix(io): fix `Audio.crop` raising when an in-memory waveform is cropped up to its very end [@MohammadHijjawi97](https://github.com/MohammadHijjawi97)
- feat(cli): add support for multi-objective pipeline optimization
- feat(cli): add `--subfolder` option to `benchmark` command [@antoinelaurent](https://github.com/antoinelaurent)
- fix(utils): fix `SpeakerDiarizationMixin.optimal_mapping` `reference` is a mapping [@Kayvan-Zahiri](https://github.com/Kayvan-Zahiri)
- fix(task): fix `balance` option crashing training with `KeyError: 'metadata'` [@MohammadHijjawi97](https://github.com/MohammadHijjawi97)
+2 -1
View File
@@ -194,7 +194,8 @@ set_telemetry_metrics(False, save_choice_as_default=True)
- 2024-04-05 > [Offline speaker diarization (speaker-diarization-3.1)](tutorials/community/offline_usage_speaker_diarization.ipynb) by [Simon Ottenhaus](https://github.com/simonottenhauskenbun)
- 2024-09-24 > [Evaluating `pyannote` pretrained speech separation pipelines](tutorials/community/eval_separation_pipeline.ipynb) by [Clément Pagés](https://github.com/)
- Tutorials
*Those tutorials were written for older versions of pyannote.audio and should be updated. Interested in working for pyannoteAI as a community manager or developer advocate? This might be a nice place to start!*
*The multi-objective tutorial targets the current version. The others were written for older versions of pyannote.audio and should be updated. Interested in working for pyannoteAI as a community manager or developer advocate? This might be a nice place to start!*
- [Finding the purity/coverage Pareto front of Community-1](tutorials/multi_objective_optimization.ipynb)
- [Applying a pretrained pipeline](tutorials/applying_a_pipeline.ipynb)
- [Adapting a pretrained pipeline to your own data](tutorials/adapting_pretrained_pipeline.ipynb)
- [Training a pipeline](tutorials/voice_activity_detection.ipynb)
+4 -4
View File
@@ -16,10 +16,10 @@ dependencies = [
"opentelemetry-api>=1.34.0",
"opentelemetry-sdk>=1.34.0",
"opentelemetry-exporter-otlp>=1.34.0",
"pyannote-core>=6.0.1",
"pyannote-core>=6.1.0",
"pyannote-database>=6.1.1",
"pyannote-metrics>=4.0.0",
"pyannote-pipeline>=4.0.0",
"pyannote-metrics>=4.2.0",
"pyannote-pipeline>=4.1.0",
"pytorch-metric-learning>=2.8.1",
"rich>=13.9.4",
"safetensors>=0.5.2",
@@ -38,7 +38,7 @@ pyannote-audio = "pyannote.audio.__main__:app"
[project.optional-dependencies]
test = ["papermill>=2.6.0", "pytest>=8.3.4"]
doc = ["sphinx-rtd-theme>=3.0.2", "sphinx>=8.1.3"]
cli = ["typer>=0.15.1"]
cli = ["filelock>=3.18.0", "typer>=0.15.1"]
separation = ["asteroid>=0.5.2", "transformers>=4.48.3"]
dev = ["ipython>=8.32.0"]
+324 -24
View File
@@ -22,11 +22,13 @@
# SOFTWARE.
import json
import os
import sys
import tempfile
import time
import types
from contextlib import nullcontext
from datetime import datetime
from datetime import UTC, datetime
from enum import Enum
from pathlib import Path
from typing import Annotated
@@ -36,13 +38,21 @@ import pyannote.database
import torch
import typer
import yaml
from filelock import FileLock
from pyannote.core import Annotation
from pyannote.metrics.base import BaseMetric
from pyannote.metrics.diarization import DiarizationErrorRate, JaccardErrorRate
from pyannote.metrics.diarization import (
DiarizationCoverage,
DiarizationErrorRate,
DiarizationPurity,
DiarizationSpeakerCountError,
JaccardErrorRate,
)
from pyannote.pipeline.optimizer import Optimizer
from rich.progress import track
from pyannote.audio import Audio, Model, Pipeline
from pyannote.audio.core.pipeline import expand_subfolders
class Subset(str, Enum):
@@ -66,15 +76,34 @@ class NumSpeakers(str, Enum):
class Metric(str, Enum):
DiarizationErrorRate = "DiarizationErrorRate"
JaccardErrorRate = "JaccardErrorRate"
DiarizationPurity = "DiarizationPurity"
DiarizationCoverage = "DiarizationCoverage"
DiarizationSpeakerCountError = "DiarizationSpeakerCountError"
@classmethod
def from_str(cls, metric: str):
def from_str(cls, metric: str) -> BaseMetric:
"""Convert a string to a Metric enum value."""
if metric == "DiarizationErrorRate":
metric = cls(metric)
if metric == cls.DiarizationErrorRate:
return DiarizationErrorRate()
elif metric == "JaccardErrorRate":
if metric == cls.JaccardErrorRate:
return JaccardErrorRate()
if metric == cls.DiarizationPurity:
return DiarizationPurity()
if metric == cls.DiarizationCoverage:
return DiarizationCoverage()
if metric == cls.DiarizationSpeakerCountError:
return DiarizationSpeakerCountError()
raise ValueError(f"Unsupported metric: {metric}")
@property
def direction(self) -> str:
"""Return the optimization direction of this metric."""
if self in {self.DiarizationPurity, self.DiarizationCoverage}:
return "maximize"
return "minimize"
def parse_device(device: Device) -> torch.device:
@@ -104,9 +133,141 @@ def get_diarization(prediction) -> Annotation:
raise ValueError("Could not find speaker diarization in prediction.")
class DiarizationMetricAdapter:
"""Apply a diarization metric to legacy and rich pipeline outputs."""
def __init__(self, metric: BaseMetric):
self.metric = metric
self.name = metric.name
def __call__(self, reference, hypothesis, **kwargs):
return self.metric(reference, get_diarization(hypothesis), **kwargs)
def __abs__(self):
return abs(self.metric)
def confidence_interval(self, **kwargs):
return self.metric.confidence_interval(**kwargs)
app = typer.Typer()
def resolve_pipeline_config(
pipeline: Path,
metric: Metric | None = None,
) -> dict:
"""Resolve a local pipeline configuration and select a Pareto trial if needed.
Parameters
----------
pipeline : Path
Path to a local pipeline YAML file, including a Pareto manifest.
metric : Metric, optional
Preferred objective when selecting a trial from a Pareto manifest.
Defaults to the first objective in the manifest. Ties are resolved by
the remaining objectives in manifest order, then by trial number.
Has no effect on single-objective configurations.
Returns
-------
config : dict
Configuration ready to serialize as YAML or pass directly to
Pipeline.from_pretrained. No file is written by this function.
Notes
-----
References such as ``$model/segmentation`` are expanded into checkpoint
dictionaries whose ``checkpoint`` is the absolute path to the source
YAML's directory and whose ``subfolder`` is ``segmentation``. Any revision
specified in the reference is preserved.
"""
# load the original pipeline configuration
path = Path(pipeline)
with open(path) as fp:
config = yaml.safe_load(fp)
# keep optimization provenance, without carrying the whole Pareto front
# into the configuration of the selected pipeline
optimization = config.pop("optimization", {})
status = optimization.pop("status", {})
metadata = {
key: value
for key, value in (optimization | status).items()
if key
in (
"metric",
"protocol",
"subset",
"trial",
"values",
"best_loss",
"last_updated",
)
}
# older single-objective configurations may not record the metric
metadata.setdefault("metric", None)
# a Pareto manifest contains several possible sets of parameters
if "pareto_front" in status:
metrics = [Metric(name) for name in optimization.get("metrics", [])]
front = status["pareto_front"]
if not metrics or not front:
raise typer.BadParameter(
"The Pareto manifest must contain objectives and trials."
)
# use the first optimization objective when no preference is provided
preferred = metric or metrics[0]
if preferred not in metrics:
raise typer.BadParameter(
f"Metric {preferred.value} is not in the Pareto manifest. "
f"Available metrics: {', '.join(m.value for m in metrics)}.",
param_hint="--metric",
)
# rank trials by the preferred metric, then by the remaining objectives
# and trial number to resolve ties deterministically
ordered = [preferred] + [m for m in metrics if m != preferred]
selected = min(
front,
key=lambda point: (
tuple(
# negate maximized metrics so that min handles both directions
(1 if m.direction == "minimize" else -1) * point["values"][m.value]
for m in ordered
)
+ (point["trial"],)
),
)
# replace the original parameters with those of the selected trial
# and keep its objective values for later reference
config["params"] = selected["params"]
metadata.update(
metric=preferred.value,
trial=selected["trial"],
values=selected["values"],
)
typer.echo(
f"Selected trial {selected['trial']} using {preferred.value}: "
+ ", ".join(
f"{name}={value:g}" for name, value in selected["values"].items()
),
err=True,
)
# resolve $model/... against the original YAML directory so that model
# locations are preserved when the config is moved or passed as a dictionary
expand_subfolders(config, str(path.parent.resolve()))
# record when this configuration was resolved and where it came from
metadata.update(
date=datetime.now(UTC).isoformat(),
source=str(path.resolve()),
)
config["optimization"] = metadata
return config
@app.command("optimize")
def optimize(
pipeline: Annotated[
@@ -152,12 +313,12 @@ def optimize(
NumSpeakers, typer.Option(help="Number of speakers (oracle or auto)")
] = NumSpeakers.AUTO,
metric: Annotated[
Metric,
list[Metric],
typer.Option(
help="Metric to optimize against",
help="Metric to optimize against. Repeat for multi-objective optimization.",
case_sensitive=False,
),
] = Metric.DiarizationErrorRate,
] = [Metric.DiarizationErrorRate],
average_case: Annotated[
bool,
typer.Option(help="Optimize for average case rather than worst case."),
@@ -167,6 +328,11 @@ def optimize(
Optimize a PIPELINE
"""
if not metric:
raise typer.BadParameter("At least one metric must be provided.")
if len(metric) != len(set(metric)):
raise typer.BadParameter("Each optimization metric must be unique.")
# load pipeline configuration file in memory. this will
# be dumped later to disk with optimized parameters
with open(pipeline) as fp:
@@ -202,23 +368,42 @@ def optimize(
getattr(loaded_protocol, subset.value)()
)
# update `get_metric` method to return the requested metric instance
# update `get_metric` and `get_direction` methods to return one value per
# requested objective
def _get_metric(self):
return Metric.from_str(metric)
metrics = tuple(
DiarizationMetricAdapter(Metric.from_str(current_metric))
for current_metric in metric
)
return metrics if len(metrics) > 1 else metrics[0]
def _get_direction(self):
directions = tuple(current_metric.direction for current_metric in metric)
return directions if len(directions) > 1 else directions[0]
optimized_pipeline.get_metric = types.MethodType(_get_metric, optimized_pipeline)
optimized_pipeline.get_direction = types.MethodType(
_get_direction, optimized_pipeline
)
# setting study name to this allows to store multiple optimizations
# for the same pipeline in the same database
study_name = f"{protocol}.{subset.value}"
# add suffix if we are using oracle number of speakers
if num_speakers == NumSpeakers.ORACLE:
study_name += ".OracleNumSpeakers"
# add suffix with optimized metric(s)
study_name += f".{'+'.join(current_metric.value for current_metric in metric)}"
# journal file to store optimization results
# if pipeline path is "config.yml", it will be stored in "config.journal"
journal = pipeline.with_suffix(".journal")
if len(metric) > 1:
result: Path = pipeline.with_suffix(f".{study_name}.pareto.yaml")
else:
result: Path = pipeline.with_suffix(f".{study_name}.yaml")
optimizer = Optimizer(
@@ -230,17 +415,76 @@ def optimize(
average_case=average_case,
)
try:
warm_start = optimized_pipeline.default_parameters()
except NotImplementedError:
warm_start = None
if optimizer.multi_objective:
previous_pareto_trials: set[int] = set()
iterations = optimizer.tune_iter(files, warm_start=warm_start)
for i, status in enumerate(iterations):
pareto_front = status["pareto_front"]
pareto_trials = {point["number"] for point in pareto_front}
if pareto_trials != previous_pareto_trials:
with FileLock(str(result) + ".lock"):
pareto_front = optimizer.pareto_front
pareto_trials = {point["number"] for point in pareto_front}
serialized_front = []
for point in pareto_front:
serialized_front.append(
{
"trial": point["number"],
"values": {
current_metric.value: float(value)
for current_metric, value in zip(
metric, point["values"]
)
},
"params": point["params"],
}
)
original_config["optimization"] = {
"protocol": protocol,
"subset": subset.value,
"metrics": [current_metric.value for current_metric in metric],
"status": {
"pareto_front": serialized_front,
"last_updated": datetime.now().isoformat(),
},
}
temporary_result = None
try:
with tempfile.NamedTemporaryFile(
mode="w",
dir=result.parent,
prefix=f".{result.name}.",
suffix=".tmp",
delete=False,
) as fp:
temporary_result = Path(fp.name)
yaml.dump(original_config, fp)
os.replace(temporary_result, result)
finally:
if temporary_result is not None:
temporary_result.unlink(missing_ok=True)
previous_pareto_trials = pareto_trials
if max_iterations and i + 1 >= max_iterations:
break
return
direction = 1 if optimized_pipeline.get_direction() == "minimize" else -1
# read best loss so far
global_best_loss: float = optimizer.best_loss
local_best_loss: float = global_best_loss
try:
warm_start = optimized_pipeline.default_parameters()
except NotImplementedError:
warm_start = None
iterations = optimizer.tune_iter(files, warm_start=warm_start)
# TODO: use pipeline.default_params() as warm_start?
@@ -260,6 +504,7 @@ def optimize(
# if we have a new global best loss, save it to disk
original_config["params"] = status["params"]
original_config["optimization"] = {
"metric": metric[0].value,
"protocol": protocol,
"subset": subset.value,
"status": {
@@ -503,18 +748,74 @@ def benchmark(
per_file: Annotated[
bool, typer.Option(help="Save one RTTM/JSON file per processed audio file.")
] = False,
metric: Annotated[
list[Metric] | None,
typer.Option(
help="Preferred Pareto objective (defaults to the first objective). Repeat to benchmark each selected configuration. Explicit selections save results in OptimizedFor<Metric> subdirectories.",
case_sensitive=False,
),
] = None,
):
"""
Benchmark a pretrained diarization PIPELINE
This will run the pipeline on all files in the specified protocol and subset,
save the results in RTTM format, and compute the Diarization Error Rate (DER)
for each file.
for each file. Repeated `--metric` options select separate configurations from
a Pareto manifest; each is evaluated with the same metrics. Explicit metric
selections save results in OptimizedFor<Metric> subdirectories of INTO.
"""
# load pretrained pipeline
objectives = list(dict.fromkeys(metric)) if metric else [None]
for objective in objectives:
output_dir = (
into if objective is None else into / f"OptimizedFor{objective.value}"
)
output_dir.mkdir(exist_ok=True)
_benchmark(
pipeline=pipeline,
protocol=protocol,
into=output_dir,
subset=subset,
revision=revision,
subfolder=subfolder,
token=token,
cache=cache,
device=device,
registry=registry,
num_speakers=num_speakers,
progress=progress,
per_file=per_file,
metric=objective,
)
def _benchmark(
pipeline: str,
protocol: str,
into: Path,
subset: Subset,
revision: str | None,
subfolder: str | None,
token: str | None,
cache: Path | None,
device: Device,
registry: Path | None,
num_speakers: NumSpeakers,
progress: bool,
per_file: bool,
metric: Metric | None,
):
benchmark_name = f"{protocol}.{subset.value}"
if num_speakers == NumSpeakers.ORACLE:
benchmark_name += ".OracleNumSpeakers"
output_config = into / f"{benchmark_name}.config.yaml"
checkpoint: str | dict = pipeline
if Path(pipeline).is_file():
checkpoint = resolve_pipeline_config(Path(pipeline), metric=metric)
pretrained_pipeline = Pipeline.from_pretrained(
pipeline,
checkpoint,
revision=revision,
subfolder=subfolder,
token=token,
@@ -556,16 +857,15 @@ def benchmark(
)
skip_metric = True
# `benchmark_name` is used as prefix to output files
benchmark_name = f"{protocol}.{subset.value}"
if num_speakers == NumSpeakers.ORACLE:
benchmark_name += ".OracleNumSpeakers"
if Path(pipeline).is_file():
with open(output_config, "x") as fp:
fp.write(f"# created_at: {checkpoint['optimization']['date']}\n")
yaml.safe_dump(checkpoint, fp)
# used to store raw predictions in JSON format
serialized_predictions: dict[str, dict] = {}
if not skip_metric:
# initialize diarization error rate metric
metric = DiarizationErrorRate()
# speaker count confusion matrix
@@ -619,7 +919,7 @@ def benchmark(
# compute metric when possible
if not skip_metric:
_ = metric(
metric(
file["annotation"],
speaker_diarization,
uem=file.get("annotated", None),
+124
View File
@@ -0,0 +1,124 @@
# Copyright (c) 2026- pyannoteAI
from types import SimpleNamespace
import pandas as pd
import pytest
import yaml
from pyannote.core import Annotation, Segment
from typer.testing import CliRunner
from pyannote.audio import __main__ as cli
@pytest.mark.parametrize(
"selected",
[[], ["DiarizationCoverage"], ["DiarizationPurity", "DiarizationCoverage"]],
)
@pytest.mark.parametrize("per_file", [False, True])
def test_benchmark_configurations(tmp_path, monkeypatch, selected, per_file):
reference = Annotation(uri="file")
reference[Segment(0, 2)] = "speaker"
files = [
{"uri": "file", "annotation": reference, "annotated": reference.get_timeline()}
]
loaded = []
calls = []
class Pipeline:
def __init__(self, config):
self.trial = config["params"]["trial"]
loaded.append(self.trial)
def to(self, device):
return self
def __call__(self, files):
for file in files:
calls.append(self.trial)
prediction = (
reference.copy() if self.trial == 0 else Annotation(uri="file")
)
yield file, SimpleNamespace(speaker_diarization=prediction)
pipeline = tmp_path / "pipeline.yaml"
pipeline.write_text(
yaml.safe_dump(
{
"pipeline": "mock",
"optimization": {
"metrics": ["DiarizationPurity", "DiarizationCoverage"],
"status": {
"pareto_front": [
{
"trial": 0,
"params": {"trial": 0},
"values": {
"DiarizationPurity": 1.0,
"DiarizationCoverage": 0.5,
},
},
{
"trial": 1,
"params": {"trial": 1},
"values": {
"DiarizationPurity": 0.5,
"DiarizationCoverage": 1.0,
},
},
]
},
},
}
)
)
monkeypatch.setattr(
cli.Pipeline, "from_pretrained", lambda config, **kw: Pipeline(config)
)
monkeypatch.setattr(
cli.pyannote.database.registry,
"get_protocol",
lambda *a, **kw: SimpleNamespace(test=lambda: iter(files)),
)
monkeypatch.setattr(cli.Audio, "get_duration", lambda self, file: 2.0)
result = CliRunner().invoke(
cli.app,
[
"benchmark",
str(pipeline),
"Debug.SpeakerDiarization.Debug",
str(tmp_path),
"--device",
"cpu",
*[arg for name in selected for arg in ("--metric", name)],
*(["--per-file"] if per_file else []),
],
)
assert result.exit_code == 0, (result.output, result.exception)
expected_trials = [
0 if name == "DiarizationPurity" else 1
for name in selected or ["DiarizationPurity"]
]
assert loaded == calls == expected_trials
for name, trial in zip(selected or ["DiarizationPurity"], expected_trials):
stem = "Debug.SpeakerDiarization.Debug.test"
output_dir = tmp_path / f"OptimizedFor{name}" if selected else tmp_path
config = yaml.safe_load((output_dir / f"{stem}.config.yaml").read_text())
assert config["params"]["trial"] == trial
report = pd.read_csv(output_dir / f"{stem}.csv", header=[0, 1], index_col=0)
assert report.loc["TOTAL", ("diarization error rate", "%")] == 100.0 * trial
assert "diarization error rate" in (output_dir / f"{stem}.txt").read_text()
assert (output_dir / f"{stem}.SpeakerCount.csv").exists()
assert (output_dir / f"{stem}.yml").exists()
rttm = (
output_dir / stem / "rttm" / "file.rttm"
if per_file
else output_dir / f"{stem}.rttm"
)
assert rttm.exists()
if selected:
assert {path.name for path in tmp_path.iterdir()} == {
"pipeline.yaml",
*(f"OptimizedFor{name}" for name in selected),
}
assert not list(tmp_path.rglob("*OptimizedMinDurationOff*"))
+317
View File
@@ -0,0 +1,317 @@
# MIT License
#
# Copyright (c) 2026- pyannoteAI
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from types import SimpleNamespace
import pytest
import yaml
from pyannote.core import Annotation, Segment
from pyannote.pipeline import Pipeline
from pyannote.pipeline.optimizer import Optimizer
from pyannote.pipeline.parameter import Integer
from typer.testing import CliRunner
from pyannote.audio.__main__ import Metric, app
from pyannote.audio.__main__ import Pipeline as AudioPipeline
class MockPipeline(Pipeline):
"""Minimal pipeline exercising the real CLI/optimizer integration."""
def __init__(self):
super().__init__()
self.threshold = Integer(0, 1)
def __call__(self, current_file, **kwargs):
return SimpleNamespace(speaker_diarization=current_file["annotation"])
def default_parameters(self):
return {"threshold": 0}
def to(self, device):
return self
class MockProtocol:
def development(self):
annotation = Annotation(uri="file")
annotation[Segment(0.0, 1.0)] = "speaker"
yield {
"uri": "file",
"annotation": annotation,
"annotated": annotation.get_timeline(),
}
def test_optimize_cli_with_single_objective(tmp_path, monkeypatch):
"""Scalar CLI optimization remains compatible with the new Optimizer API."""
pipeline_yml = tmp_path / "pipeline.yaml"
pipeline_yml.write_text("pipeline: mock\n")
mock_pipeline = MockPipeline()
monkeypatch.setattr(
AudioPipeline,
"from_pretrained",
lambda *args, **kwargs: mock_pipeline,
)
monkeypatch.setattr(
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
lambda *args, **kwargs: MockProtocol(),
)
runner = CliRunner()
result = runner.invoke(
app,
[
"optimize",
str(pipeline_yml),
"Debug.SpeakerDiarization.Debug",
"--subset",
"development",
"--device",
"cpu",
"--max-iterations",
"1",
"--average-case",
],
)
assert result.exit_code == 0, result.output
optimized_yml = tmp_path / (
"pipeline.Debug.SpeakerDiarization.Debug.development.DiarizationErrorRate.yaml"
)
optimized = yaml.safe_load(optimized_yml.read_text())
assert optimized["params"] == {"threshold": 0}
assert optimized["optimization"]["status"]["best_loss"] == 0.0
def test_optimize_cli_with_multiple_objectives(tmp_path, monkeypatch):
"""Repeated metric options produce a named Pareto-front manifest."""
pipeline_yml = tmp_path / "pipeline.yaml"
pipeline_yml.write_text("pipeline: mock\n")
mock_pipeline = MockPipeline()
monkeypatch.setattr(
AudioPipeline,
"from_pretrained",
lambda *args, **kwargs: mock_pipeline,
)
monkeypatch.setattr(
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
lambda *args, **kwargs: MockProtocol(),
)
runner = CliRunner()
result = runner.invoke(
app,
[
"optimize",
str(pipeline_yml),
"Debug.SpeakerDiarization.Debug",
"--subset",
"development",
"--device",
"cpu",
"--max-iterations",
"1",
"--metric",
"DiarizationPurity",
"--metric",
"DiarizationCoverage",
"--average-case",
],
)
assert result.exit_code == 0, result.output
pareto_yml = tmp_path / (
"pipeline.Debug.SpeakerDiarization.Debug.development."
"DiarizationPurity+DiarizationCoverage.pareto.yaml"
)
pareto = yaml.safe_load(pareto_yml.read_text())["optimization"]
assert pareto["metrics"] == ["DiarizationPurity", "DiarizationCoverage"]
assert pareto["status"]["pareto_front"] == [
{
"trial": 0,
"values": {
"DiarizationPurity": 1.0,
"DiarizationCoverage": 1.0,
},
"params": {"threshold": 0},
}
]
def test_multi_objective_export_refreshes_pareto_front(tmp_path, monkeypatch):
pipeline_yml = tmp_path / "pipeline.yaml"
pipeline_yml.write_text("pipeline: mock\n")
monkeypatch.setattr(
AudioPipeline, "from_pretrained", lambda *args, **kwargs: MockPipeline()
)
monkeypatch.setattr(
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
lambda *args, **kwargs: MockProtocol(),
)
monkeypatch.setattr(
Optimizer,
"tune_iter",
lambda self, *args, **kwargs: iter(
[
{
"pareto_front": [
{
"number": 0,
"values": (0.5, 0.5),
"params": {"threshold": 0},
}
]
}
]
),
)
monkeypatch.setattr(
Optimizer,
"pareto_front",
property(
lambda self: [
{
"number": 1,
"values": (0.25, 0.75),
"params": {"threshold": 1},
}
]
),
)
result = CliRunner().invoke(
app,
[
"optimize",
str(pipeline_yml),
"Debug.SpeakerDiarization.Debug",
"--subset",
"development",
"--device",
"cpu",
"--max-iterations",
"1",
"--metric",
"DiarizationPurity",
"--metric",
"DiarizationCoverage",
"--average-case",
],
)
assert result.exit_code == 0, result.output
output = tmp_path / (
"pipeline.Debug.SpeakerDiarization.Debug.development."
"DiarizationPurity+DiarizationCoverage.pareto.yaml"
)
pareto = yaml.safe_load(output.read_text())["optimization"]["status"][
"pareto_front"
]
assert pareto == [
{
"trial": 1,
"values": {"DiarizationPurity": 0.25, "DiarizationCoverage": 0.75},
"params": {"threshold": 1},
}
]
assert (tmp_path / f"{output.name}.lock").exists()
assert not list(tmp_path.glob(f".{output.name}.*.tmp"))
@pytest.mark.parametrize("multi_objective", [False, True])
def test_optimize_cli_with_speaker_count_error(tmp_path, monkeypatch, multi_objective):
"""Count errors respect UEM and average files equally in both optimization modes."""
class SpeakerCountPipeline(MockPipeline):
def __call__(self, current_file, **kwargs):
return SimpleNamespace(speaker_diarization=current_file["hypothesis"])
class SpeakerCountProtocol:
def development(self):
for index, (duration, extra_speakers) in enumerate([(1.0, 1), (10.0, 3)]):
reference = Annotation(uri=f"file{index}")
reference[Segment(0.0, duration)] = "speaker"
hypothesis = reference.copy()
for speaker in range(extra_speakers):
hypothesis[Segment(0.0, duration), speaker + 1] = f"extra{speaker}"
hypothesis[Segment(duration, duration + 1.0)] = "outside_uem"
yield {
"uri": reference.uri,
"annotation": reference,
"annotated": reference.get_timeline(),
"hypothesis": hypothesis,
}
pipeline_yml = tmp_path / "pipeline.yaml"
pipeline_yml.write_text("pipeline: mock\n")
mock_pipeline = SpeakerCountPipeline()
monkeypatch.setattr(
AudioPipeline, "from_pretrained", lambda *args, **kwargs: mock_pipeline
)
monkeypatch.setattr(
"pyannote.audio.__main__.pyannote.database.registry.get_protocol",
lambda *args, **kwargs: SpeakerCountProtocol(),
)
metrics = ["DiarizationSpeakerCountError"]
if multi_objective:
metrics.append("DiarizationCoverage")
metric_options = [option for metric in metrics for option in ("--metric", metric)]
result = CliRunner().invoke(
app,
[
"optimize",
str(pipeline_yml),
"Debug.SpeakerDiarization.Debug",
"--device",
"cpu",
"--max-iterations",
"1",
"--average-case",
*metric_options,
],
)
assert result.exit_code == 0, result.output
assert Metric.DiarizationSpeakerCountError.direction == "minimize"
suffix = ".pareto.yaml" if multi_objective else ".yaml"
output = tmp_path / (
"pipeline.Debug.SpeakerDiarization.Debug.development."
+ "+".join(metrics)
+ suffix
)
optimization = yaml.safe_load(output.read_text())["optimization"]
if multi_objective:
assert mock_pipeline.get_direction() == ("minimize", "maximize")
assert optimization["metrics"] == metrics
assert optimization["status"]["pareto_front"][0]["values"] == {
"DiarizationSpeakerCountError": 2.0,
"DiarizationCoverage": 1.0,
}
else:
assert optimization["status"]["best_loss"] == 2.0
@@ -0,0 +1,252 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Multi-objective optimization of Community-1\n",
"\n",
"Speaker diarization purity and coverage usually conflict: splitting speakers into more clusters tends to improve purity, while merging clusters tends to improve coverage. This tutorial uses `pyannote-audio optimize` to tune `pyannote/speaker-diarization-community-1` for both metrics and retrieves the Pareto front—the operating points for which neither metric can improve without degrading the other. It then exports configurations for the best purity, best coverage, and best harmonic mean of both.\n",
"\n",
"Install `pyannote.audio` with its `cli` extra in the notebook's Python environment. You need a Hugging Face token with access to Community-1 (log in beforehand or set `HF_TOKEN`) and a [`pyannote.database`](https://github.com/pyannote/pyannote-database) protocol with reference annotations."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from copy import deepcopy\n",
"from pathlib import Path\n",
"import subprocess\n",
"import sys\n",
"\n",
"import matplotlib.pyplot as plt\n",
"import yaml\n",
"from huggingface_hub import snapshot_download\n",
"from pyannote.audio import Pipeline\n",
"from pyannote.audio.core.pipeline import expand_subfolders"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Prepare Community-1 and the development protocol\n",
"\n",
"Download Community-1 into a writable local directory, including its model subfolders. The CLI takes a local pipeline configuration and writes the optimization journal and results beside it. If you already have a complete local copy, set `pipeline_config` to its `config.yaml` and skip the download."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model_directory = Path(\"community1\").resolve()\n",
"snapshot_download(\n",
" \"pyannote/speaker-diarization-community-1\",\n",
" local_dir=model_directory,\n",
")\n",
"pipeline_config = model_directory / \"config.yaml\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Replace the database configuration and protocol names below. Optimize on the development set; reserve the final test set for evaluation."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"database_registry = Path(\"/path/to/database.yml\")\n",
"protocol_name = \"MyDatabase.SpeakerDiarization.MyProtocol\"\n",
"subset = \"development\"\n",
"metrics = [\"DiarizationPurity\", \"DiarizationCoverage\"]\n",
"max_iterations = 100"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Optimize both metrics with the CLI\n",
"\n",
"Repeat `--metric` to define a multi-objective study. The command below runs `pyannote-audio optimize` using the notebook's Python environment. The CLI handles metric evaluation and device selection and keeps updating a `*.pareto.yaml` manifest containing every non-dominated trial, its named metric values, and its pipeline parameters.\n",
"\n",
"`--average-case` uses purity and coverage accumulated over the whole development set. Omit it to optimize the lower 90% confidence bound of each metric instead. The journal lets an interrupted run resume and allows multiple optimization processes to share the study. Re-run with the same settings to continue the study; use a separate pipeline directory when changing evaluation settings.\n",
"\n",
"Other objectives can also be combined using repeated `--metric` options. For example, `--metric DiarizationSpeakerCountError --average-case` minimizes the mean absolute difference between predicted and reference speaker counts, weighting files equally and counting only speakers in annotated evaluation regions. This objective is measured in speakers, not percent."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"command = [\n",
" sys.executable, \"-m\", \"pyannote.audio\", \"optimize\",\n",
" str(pipeline_config), protocol_name,\n",
" \"--registry\", str(database_registry),\n",
" \"--subset\", subset,\n",
" \"--average-case\",\n",
" \"--max-iterations\", str(max_iterations),\n",
"]\n",
"for metric in metrics:\n",
" command.extend([\"--metric\", metric])\n",
"\n",
"subprocess.run(command, check=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Read and plot the Pareto front\n",
"\n",
"The manifest filename includes the protocol, subset, and metrics in command-line order. Each entry contains the trial number, a dictionary of named metric values, and the corresponding nested Community-1 parameters."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"study_name = f\"{protocol_name}.{subset}.{'+'.join(metrics)}\"\n",
"manifest_path = pipeline_config.with_suffix(f\".{study_name}.pareto.yaml\")\n",
"manifest = yaml.safe_load(manifest_path.read_text())\n",
"pareto_front = sorted(\n",
" manifest[\"optimization\"][\"status\"][\"pareto_front\"],\n",
" key=lambda point: point[\"values\"][\"DiarizationPurity\"],\n",
")\n",
"if not pareto_front:\n",
" raise ValueError(\"No Pareto trials are available. Run optimization first.\")\n",
"pareto_front"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"purity = [point[\"values\"][\"DiarizationPurity\"] for point in pareto_front]\n",
"coverage = [point[\"values\"][\"DiarizationCoverage\"] for point in pareto_front]\n",
"\n",
"fig, ax = plt.subplots()\n",
"ax.plot(purity, coverage, \"o-\")\n",
"ax.set(xlabel=\"Diarization purity\", ylabel=\"Diarization coverage\")\n",
"ax.grid(True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Select the best purity, coverage, and harmonic mean\n",
"\n",
"A multi-objective study has no single best configuration. Select three operating points from the Pareto front: maximum purity, maximum coverage, and maximum harmonic mean, `2 * purity * coverage / (purity + coverage)`. Define the harmonic mean as zero when both metrics are zero. These selections can refer to the same trial.\n",
"\n",
"For tied purity or coverage, prefer the higher other metric. For tied harmonic means, prefer higher purity, then coverage. Finally, prefer the earlier trial. With `--average-case`, these are development-set scores; otherwise, the selection uses the optimized confidence bounds."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def harmonic_mean(point):\n",
" purity = point[\"values\"][\"DiarizationPurity\"]\n",
" coverage = point[\"values\"][\"DiarizationCoverage\"]\n",
" return 2 * purity * coverage / (purity + coverage) if purity + coverage else 0.0\n",
"\n",
"\n",
"def rank(point, criterion):\n",
" purity = point[\"values\"][\"DiarizationPurity\"]\n",
" coverage = point[\"values\"][\"DiarizationCoverage\"]\n",
" score = {\n",
" \"purity\": purity,\n",
" \"coverage\": coverage,\n",
" \"harmonic_mean\": harmonic_mean(point),\n",
" }[criterion]\n",
" return score, purity, coverage, -point[\"trial\"]\n",
"\n",
"\n",
"selected_points = {\n",
" criterion: max(pareto_front, key=lambda point: rank(point, criterion))\n",
" for criterion in (\"purity\", \"coverage\", \"harmonic_mean\")\n",
"}\n",
"for criterion, point in selected_points.items():\n",
" print(\n",
" f\"Best {criterion}: trial {point['trial']}, \"\n",
" f\"purity={point['values']['DiarizationPurity']:.3f}, \"\n",
" f\"coverage={point['values']['DiarizationCoverage']:.3f}, \"\n",
" f\"harmonic mean={harmonic_mean(point):.3f}\"\n",
" )"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Return and save the three pipeline configurations\n",
"\n",
"Each configuration includes the pipeline definition, the selected parameters, and optimization provenance. Resolve model subfolders against the original model directory so the configurations can be loaded from dictionaries or YAML files elsewhere on this machine. The model files must remain available at that location.\n",
"\n",
"`best_configs` returns the three configuration dictionaries. The corresponding `config.best_purity.yaml`, `config.best_coverage.yaml`, and `config.best_harmonic_mean.yaml` files are saved beside the original configuration. Load one with, for example, `Pipeline.from_pretrained(best_config_paths[\"harmonic_mean\"])`."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"best_configs = {}\n",
"best_config_paths = {}\n",
"for criterion, point in selected_points.items():\n",
" config = deepcopy(manifest)\n",
" config[\"params\"] = deepcopy(point[\"params\"])\n",
" config[\"optimization\"] = {\n",
" \"protocol\": protocol_name,\n",
" \"subset\": subset,\n",
" \"metrics\": metrics.copy(),\n",
" \"selection\": criterion,\n",
" \"trial\": point[\"trial\"],\n",
" \"values\": deepcopy(point[\"values\"]),\n",
" \"harmonic_mean\": harmonic_mean(point),\n",
" \"source\": str(manifest_path.resolve()),\n",
" }\n",
" expand_subfolders(config, str(manifest_path.parent.resolve()))\n",
" best_configs[criterion] = config\n",
" config_path = pipeline_config.with_suffix(f\".best_{criterion}.yaml\")\n",
" config_path.write_text(yaml.safe_dump(config, sort_keys=False))\n",
" best_config_paths[criterion] = config_path\n",
"\n",
"best_configs"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}