feat(cli): add "download" and "optimize" commands (#1843)

This commit is contained in:
Hervé BREDIN authored and GitHub committed 2025-02-24 13:00:19 +01:00
1 parent b8d20cafad
commit ce495767be
4 files changed
+244 -16

No files matched your search

+1 -1
View File
@@ -39,7 +39,7 @@ Clipping and speaker/source alignment issues in speech separation pipeline have
- feat(utils): add `FilterByNumberOfSpeakers` protocol files filter
- feat(core): add `Calibration` class to calibrate logits/distances into probabilities
- feat(metric): add `DetectionErrorRate`, `SegmentationErrorRate`, `DiarizationPrecision`, and `DiarizationRecall` metrics
- feat(cli): add CLI to apply (and benchmark) pretrained pipelines
- feat(cli): add CLI to download, apply, benchmark, or optimize pipelines
### Improvements
+1 -2
View File
@@ -19,7 +19,7 @@ dependencies = [
"pyannote-core>=6.0.0rc1",
"pyannote-database>=6.0.0rc1",
"pyannote-metrics>=4.0.0rc1",
"pyannote-pipeline>=4.0.0rc1",
"pyannote-pipeline>=4.0.0rc2",
"pytorch-metric-learning>=2.8.1",
"rich>=13.9.4",
"safetensors>=0.5.2",
@@ -34,7 +34,6 @@ dependencies = [
[project.scripts]
pyannote-audio = "pyannote.audio.__main__:app"
[project.optional-dependencies]
test = [
"papermill>=2.6.0",
+237 -8
View File
@@ -23,9 +23,9 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import sys
from contextlib import nullcontext
from datetime import datetime
from enum import Enum
from pathlib import Path
from typing import Optional
@@ -33,7 +33,9 @@ from typing import Optional
import pyannote.database
import torch
import typer
import yaml
from pyannote.core import Annotation
from pyannote.pipeline.optimizer import Optimizer
from typing_extensions import Annotated
from pyannote.audio import Pipeline
@@ -52,6 +54,11 @@ class Device(str, Enum):
AUTO = "auto"
class NumSpeakers(str, Enum):
ORACLE = "oracle"
AUTO = "auto"
def parse_device(device: Device) -> torch.device:
if device == Device.AUTO:
if torch.cuda.is_available():
@@ -69,7 +76,190 @@ def parse_device(device: Device) -> torch.device:
app = typer.Typer()
# TODO: add option to download pretrained pipeline for later use without internet
@app.command("optimize")
def optimize(
pipeline: Annotated[
Path,
typer.Argument(
help="Path to pipeline YAML configuration file",
exists=True,
dir_okay=False,
file_okay=True,
writable=True,
resolve_path=True,
),
],
protocol: Annotated[
str,
typer.Argument(help="Protocol used for optimization"),
],
subset: Annotated[
Subset,
typer.Option(
help="Subset used for optimization",
case_sensitive=False,
),
] = Subset.development,
device: Annotated[
Device, typer.Option(help="Accelerator to use (CPU, CUDA, MPS)")
] = Device.AUTO,
registry: Annotated[
Optional[Path],
typer.Option(
help="Loaded registry",
exists=True,
dir_okay=False,
file_okay=True,
readable=True,
),
] = None,
max_iterations: Annotated[
Optional[int],
typer.Option(help="Number of iterations to run. Defaults to run indefinitely."),
] = None,
num_speakers: Annotated[
NumSpeakers, typer.Option(help="Number of speakers (oracle or auto)")
] = NumSpeakers.AUTO,
):
"""
Optimize a PIPELINE
"""
# load pipeline configuration file in memory. this will
# be dumped later to disk with optimized parameters
with open(pipeline, "r") as fp:
original_config = yaml.load(fp, Loader=yaml.SafeLoader)
# load pipeline
optimized_pipeline = Pipeline.from_pretrained(pipeline)
if optimized_pipeline is None:
raise ValueError(f"Could not load pipeline from {pipeline}")
# send pipeline to device
torch_device = parse_device(device)
optimized_pipeline.to(torch_device)
# load protocol from (optional) registry
if registry:
pyannote.database.registry.load_database(registry)
preprocessors = {"audio": pyannote.database.FileFinder()}
# pass number of speakers to pipeline if requested
if num_speakers == NumSpeakers.ORACLE:
preprocessors["pipeline_kwargs"] = lambda protocol_file: {
"num_speakers": len(protocol_file["annotation"].labels())
}
loaded_protocol = pyannote.database.registry.get_protocol(
protocol, preprocessors=preprocessors
)
files: list[pyannote.database.ProtocolFile] = list(
getattr(loaded_protocol, subset.value)()
)
# setting study name to this allows to store multiple optimizations
# for the same pipeline in the same database
study_name = f"{protocol}.{subset.value}"
# add suffix if we are using oracle number of speakers
if num_speakers == NumSpeakers.ORACLE:
study_name += ".OracleNumSpeakers"
# journal file to store optimization results
# if pipeline path is "config.yml", it will be stored in "config.journal"
journal = pipeline.with_suffix(".journal")
result: Path = pipeline.with_suffix(f".{study_name}.yaml")
optimizer = Optimizer(
optimized_pipeline,
db=journal,
study_name=study_name,
sampler=None, # TODO: support sampler
pruner=None, # TODO: support pruner
average_case=False,
)
direction = 1 if optimized_pipeline.get_direction() == "minimize" else -1
# read best loss so far
global_best_loss: float = optimizer.best_loss
local_best_loss: float = global_best_loss
#
try:
warm_start = optimized_pipeline.default_parameters()
except NotImplementedError:
warm_start = None
iterations = optimizer.tune_iter(files, warm_start=warm_start)
# TODO: use pipeline.default_params() as warm_start?
for i, status in enumerate(iterations):
loss = status["loss"]
# check whether this iteration led to a better loss
# than all previous iterations for this run
if direction * (loss - local_best_loss) < 0:
# new (local) best loss
local_best_loss = loss
# if it did, also check directly from the central database if this is a new global best loss
# (we might have multiple optimizations going on simultaneously)
if local_best_loss == (global_best_loss := optimizer.best_loss):
# if we have a new global best loss, save it to disk
original_config["params"] = status["params"]
original_config["optimization"] = {
"protocol": protocol,
"subset": subset.value,
"status": {
"best_loss": local_best_loss,
"last_updated": datetime.now().isoformat(),
},
}
with open(result, "w") as fp:
yaml.dump(original_config, fp)
local_best_loss = global_best_loss
if max_iterations and i + 1 >= max_iterations:
break
@app.command("download")
def download(
pipeline: Annotated[
str,
typer.Argument(
help="Pretrained pipeline (e.g. pyannote/speaker-diarization-3.1)"
),
],
token: Annotated[
str,
typer.Argument(
help="Huggingface token to be used for downloading from Huggingface hub."
),
],
cache: Annotated[
Path,
typer.Option(
help="Path to the folder where files downloaded from Huggingface hub are stored.",
exists=True,
dir_okay=True,
file_okay=False,
writable=True,
resolve_path=True,
),
] = None,
):
"""
Download a pretrained PIPELINE to disk for later offline use.
"""
# load pretrained pipeline
_ = Pipeline.from_pretrained(pipeline, token=token, cache_dir=cache)
@app.command("apply")
@@ -103,13 +293,24 @@ def apply(
device: Annotated[
Device, typer.Option(help="Accelerator to use (CPU, CUDA, MPS)")
] = Device.AUTO,
cache: Annotated[
Path,
typer.Option(
help="Path to the folder where files downloaded from Huggingface hub are stored.",
exists=True,
dir_okay=True,
file_okay=False,
writable=True,
resolve_path=True,
),
] = None,
):
"""
Apply a pretrained PIPELINE to an AUDIO file
"""
# load pretrained pipeline
pretrained_pipeline = Pipeline.from_pretrained(pipeline)
pretrained_pipeline = Pipeline.from_pretrained(pipeline, cache_dir=cache)
# send pipeline to device
torch_device = parse_device(device)
@@ -166,13 +367,27 @@ def benchmark(
readable=True,
),
] = None,
num_speakers: Annotated[
NumSpeakers, typer.Option(help="Number of speakers (oracle or auto)")
] = NumSpeakers.AUTO,
cache: Annotated[
Path,
typer.Option(
help="Path to the folder where files downloaded from Huggingface hub are stored.",
exists=True,
dir_okay=True,
file_okay=False,
writable=True,
resolve_path=True,
),
] = None,
):
"""
Benchmark a pretrained PIPELINE
"""
# load pretrained pipeline
pretrained_pipeline = Pipeline.from_pretrained(pipeline)
pretrained_pipeline = Pipeline.from_pretrained(pipeline, cache_dir=cache)
# send pipeline to device
torch_device = parse_device(device)
@@ -188,13 +403,27 @@ def benchmark(
if registry:
pyannote.database.registry.load_database(registry)
preprocessors = {"audio": pyannote.database.FileFinder()}
# pass number of speakers to pipeline if requested
if num_speakers == NumSpeakers.ORACLE:
preprocessors["pipeline_kwargs"] = lambda protocol_file: {
"num_speakers": len(protocol_file["annotation"].labels())
}
loaded_protocol = pyannote.database.registry.get_protocol(
protocol, {"audio": pyannote.database.FileFinder()}
protocol, preprocessors=preprocessors
)
with open(into / f"{protocol}.{subset.value}.rttm", "w") as rttm:
benchmark_name = f"{protocol}.{subset.value}"
if num_speakers == NumSpeakers.ORACLE:
benchmark_name += ".OracleNumSpeakers"
with open(into / f"{benchmark_name}.rttm", "w") as rttm:
for file in getattr(loaded_protocol, subset.value)():
prediction: Annotation = pretrained_pipeline(file)
prediction: Annotation = pretrained_pipeline(
file, **file.get("pipeline_kwargs", {})
)
prediction.write_rttm(rttm)
rttm.flush()
@@ -211,7 +440,7 @@ def benchmark(
if metric is None:
return
with open(into / f"{protocol}.{subset.value}.txt", "w") as txt:
with open(into / f"{benchmark_name}.txt", "w") as txt:
txt.write(str(metric))
print(str(metric))
Generated
+5 -5
View File
@@ -2002,7 +2002,7 @@ wheels = [
[[package]]
name = "pyannote-audio"
version = "3.3.3.dev52+ge662b9c2.d20250212"
version = "3.3.3.dev59+g45d30114"
source = { editable = "." }
dependencies = [
{ name = "asteroid-filterbanks" },
@@ -2063,7 +2063,7 @@ requires-dist = [
{ name = "pyannote-core", specifier = ">=6.0.0rc1" },
{ name = "pyannote-database", specifier = ">=6.0.0rc1" },
{ name = "pyannote-metrics", specifier = ">=4.0.0rc1" },
{ name = "pyannote-pipeline", specifier = ">=4.0.0rc1" },
{ name = "pyannote-pipeline", specifier = ">=4.0.0rc2" },
{ name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" },
{ name = "pytorch-metric-learning", specifier = ">=2.8.1" },
{ name = "rich", specifier = ">=13.9.4" },
@@ -2130,7 +2130,7 @@ wheels = [
[[package]]
name = "pyannote-pipeline"
version = "4.0.0rc1"
version = "4.0.0rc2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "filelock" },
@@ -2140,9 +2140,9 @@ dependencies = [
{ name = "pyyaml" },
{ name = "tqdm" },
]
sdist = { url = "https://files.pythonhosted.org/packages/f1/33/e24b58807851f6ce42de0929470b354dd71983590afb4492634a34440b97/pyannote_pipeline-4.0.0rc1.tar.gz", hash = "sha256:8b1f12b3f122f4608d311ba249bab4b04020f3ee1db68f8317452a1103451100", size = 348744 }
sdist = { url = "https://files.pythonhosted.org/packages/59/5f/c3514701caef9db4b2b30c023a4f7341c71d15af1ecf5574807415650072/pyannote_pipeline-4.0.0rc2.tar.gz", hash = "sha256:ee140379669e5ade9033f3fdea0c3d7fbe865a41bfc1a17a4f1cf47102eb7c30", size = 348886 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/42/2e/d3b87cabb483e2c14b10ab799f443397d23554476284703e1eb1edb35f0f/pyannote_pipeline-4.0.0rc1-py3-none-any.whl", hash = "sha256:b8ad4fe06c96787e86d8f53ec26fadda54700c18b77656d7008497124fcb7cb8", size = 22134 },
{ url = "https://files.pythonhosted.org/packages/8c/c8/86d75670b7deab167989fca5dbb050648dd0349c6a9ddd5e97d0bc07286f/pyannote_pipeline-4.0.0rc2-py3-none-any.whl", hash = "sha256:2307b5ba2c2233f61e998a324db06e62f1c4c2517df28140cff591a894ab0c93", size = 22254 },
]
[[package]]