diff --git a/CHANGELOG.md b/CHANGELOG.md index c667125c..b02697f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,7 @@ Clipping and speaker/source alignment issues in speech separation pipeline have - feat(utils): add `FilterByNumberOfSpeakers` protocol files filter - feat(core): add `Calibration` class to calibrate logits/distances into probabilities - feat(metric): add `DetectionErrorRate`, `SegmentationErrorRate`, `DiarizationPrecision`, and `DiarizationRecall` metrics -- feat(cli): add CLI to apply (and benchmark) pretrained pipelines +- feat(cli): add CLI to download, apply, benchmark, or optimize pipelines ### Improvements diff --git a/pyproject.toml b/pyproject.toml index c413f6a5..623a364a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "pyannote-core>=6.0.0rc1", "pyannote-database>=6.0.0rc1", "pyannote-metrics>=4.0.0rc1", - "pyannote-pipeline>=4.0.0rc1", + "pyannote-pipeline>=4.0.0rc2", "pytorch-metric-learning>=2.8.1", "rich>=13.9.4", "safetensors>=0.5.2", @@ -34,7 +34,6 @@ dependencies = [ [project.scripts] pyannote-audio = "pyannote.audio.__main__:app" - [project.optional-dependencies] test = [ "papermill>=2.6.0", diff --git a/src/pyannote/audio/__main__.py b/src/pyannote/audio/__main__.py index cd6fa1fe..beeb1251 100644 --- a/src/pyannote/audio/__main__.py +++ b/src/pyannote/audio/__main__.py @@ -23,9 +23,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - import sys from contextlib import nullcontext +from datetime import datetime from enum import Enum from pathlib import Path from typing import Optional @@ -33,7 +33,9 @@ from typing import Optional import pyannote.database import torch import typer +import yaml from pyannote.core import Annotation +from pyannote.pipeline.optimizer import Optimizer from typing_extensions import Annotated from pyannote.audio import Pipeline @@ -52,6 +54,11 @@ class Device(str, Enum): AUTO = "auto" +class NumSpeakers(str, Enum): + ORACLE = "oracle" + AUTO = "auto" + + def parse_device(device: Device) -> torch.device: if device == Device.AUTO: if torch.cuda.is_available(): @@ -69,7 +76,190 @@ def parse_device(device: Device) -> torch.device: app = typer.Typer() -# TODO: add option to download pretrained pipeline for later use without internet +@app.command("optimize") +def optimize( + pipeline: Annotated[ + Path, + typer.Argument( + help="Path to pipeline YAML configuration file", + exists=True, + dir_okay=False, + file_okay=True, + writable=True, + resolve_path=True, + ), + ], + protocol: Annotated[ + str, + typer.Argument(help="Protocol used for optimization"), + ], + subset: Annotated[ + Subset, + typer.Option( + help="Subset used for optimization", + case_sensitive=False, + ), + ] = Subset.development, + device: Annotated[ + Device, typer.Option(help="Accelerator to use (CPU, CUDA, MPS)") + ] = Device.AUTO, + registry: Annotated[ + Optional[Path], + typer.Option( + help="Loaded registry", + exists=True, + dir_okay=False, + file_okay=True, + readable=True, + ), + ] = None, + max_iterations: Annotated[ + Optional[int], + typer.Option(help="Number of iterations to run. Defaults to run indefinitely."), + ] = None, + num_speakers: Annotated[ + NumSpeakers, typer.Option(help="Number of speakers (oracle or auto)") + ] = NumSpeakers.AUTO, +): + """ + Optimize a PIPELINE + """ + + # load pipeline configuration file in memory. this will + # be dumped later to disk with optimized parameters + with open(pipeline, "r") as fp: + original_config = yaml.load(fp, Loader=yaml.SafeLoader) + + # load pipeline + optimized_pipeline = Pipeline.from_pretrained(pipeline) + if optimized_pipeline is None: + raise ValueError(f"Could not load pipeline from {pipeline}") + + # send pipeline to device + torch_device = parse_device(device) + optimized_pipeline.to(torch_device) + + # load protocol from (optional) registry + if registry: + pyannote.database.registry.load_database(registry) + + preprocessors = {"audio": pyannote.database.FileFinder()} + + # pass number of speakers to pipeline if requested + if num_speakers == NumSpeakers.ORACLE: + preprocessors["pipeline_kwargs"] = lambda protocol_file: { + "num_speakers": len(protocol_file["annotation"].labels()) + } + + loaded_protocol = pyannote.database.registry.get_protocol( + protocol, preprocessors=preprocessors + ) + + files: list[pyannote.database.ProtocolFile] = list( + getattr(loaded_protocol, subset.value)() + ) + + # setting study name to this allows to store multiple optimizations + # for the same pipeline in the same database + study_name = f"{protocol}.{subset.value}" + # add suffix if we are using oracle number of speakers + if num_speakers == NumSpeakers.ORACLE: + study_name += ".OracleNumSpeakers" + + # journal file to store optimization results + # if pipeline path is "config.yml", it will be stored in "config.journal" + journal = pipeline.with_suffix(".journal") + + result: Path = pipeline.with_suffix(f".{study_name}.yaml") + + optimizer = Optimizer( + optimized_pipeline, + db=journal, + study_name=study_name, + sampler=None, # TODO: support sampler + pruner=None, # TODO: support pruner + average_case=False, + ) + + direction = 1 if optimized_pipeline.get_direction() == "minimize" else -1 + + # read best loss so far + global_best_loss: float = optimizer.best_loss + local_best_loss: float = global_best_loss + + # + try: + warm_start = optimized_pipeline.default_parameters() + except NotImplementedError: + warm_start = None + + iterations = optimizer.tune_iter(files, warm_start=warm_start) + + # TODO: use pipeline.default_params() as warm_start? + + for i, status in enumerate(iterations): + loss = status["loss"] + + # check whether this iteration led to a better loss + # than all previous iterations for this run + if direction * (loss - local_best_loss) < 0: + # new (local) best loss + local_best_loss = loss + + # if it did, also check directly from the central database if this is a new global best loss + # (we might have multiple optimizations going on simultaneously) + if local_best_loss == (global_best_loss := optimizer.best_loss): + # if we have a new global best loss, save it to disk + original_config["params"] = status["params"] + original_config["optimization"] = { + "protocol": protocol, + "subset": subset.value, + "status": { + "best_loss": local_best_loss, + "last_updated": datetime.now().isoformat(), + }, + } + with open(result, "w") as fp: + yaml.dump(original_config, fp) + + local_best_loss = global_best_loss + + if max_iterations and i + 1 >= max_iterations: + break + + +@app.command("download") +def download( + pipeline: Annotated[ + str, + typer.Argument( + help="Pretrained pipeline (e.g. pyannote/speaker-diarization-3.1)" + ), + ], + token: Annotated[ + str, + typer.Argument( + help="Huggingface token to be used for downloading from Huggingface hub." + ), + ], + cache: Annotated[ + Path, + typer.Option( + help="Path to the folder where files downloaded from Huggingface hub are stored.", + exists=True, + dir_okay=True, + file_okay=False, + writable=True, + resolve_path=True, + ), + ] = None, +): + """ + Download a pretrained PIPELINE to disk for later offline use. + """ + + # load pretrained pipeline + _ = Pipeline.from_pretrained(pipeline, token=token, cache_dir=cache) @app.command("apply") @@ -103,13 +293,24 @@ def apply( device: Annotated[ Device, typer.Option(help="Accelerator to use (CPU, CUDA, MPS)") ] = Device.AUTO, + cache: Annotated[ + Path, + typer.Option( + help="Path to the folder where files downloaded from Huggingface hub are stored.", + exists=True, + dir_okay=True, + file_okay=False, + writable=True, + resolve_path=True, + ), + ] = None, ): """ Apply a pretrained PIPELINE to an AUDIO file """ # load pretrained pipeline - pretrained_pipeline = Pipeline.from_pretrained(pipeline) + pretrained_pipeline = Pipeline.from_pretrained(pipeline, cache_dir=cache) # send pipeline to device torch_device = parse_device(device) @@ -166,13 +367,27 @@ def benchmark( readable=True, ), ] = None, + num_speakers: Annotated[ + NumSpeakers, typer.Option(help="Number of speakers (oracle or auto)") + ] = NumSpeakers.AUTO, + cache: Annotated[ + Path, + typer.Option( + help="Path to the folder where files downloaded from Huggingface hub are stored.", + exists=True, + dir_okay=True, + file_okay=False, + writable=True, + resolve_path=True, + ), + ] = None, ): """ Benchmark a pretrained PIPELINE """ # load pretrained pipeline - pretrained_pipeline = Pipeline.from_pretrained(pipeline) + pretrained_pipeline = Pipeline.from_pretrained(pipeline, cache_dir=cache) # send pipeline to device torch_device = parse_device(device) @@ -188,13 +403,27 @@ def benchmark( if registry: pyannote.database.registry.load_database(registry) + preprocessors = {"audio": pyannote.database.FileFinder()} + + # pass number of speakers to pipeline if requested + if num_speakers == NumSpeakers.ORACLE: + preprocessors["pipeline_kwargs"] = lambda protocol_file: { + "num_speakers": len(protocol_file["annotation"].labels()) + } + loaded_protocol = pyannote.database.registry.get_protocol( - protocol, {"audio": pyannote.database.FileFinder()} + protocol, preprocessors=preprocessors ) - with open(into / f"{protocol}.{subset.value}.rttm", "w") as rttm: + benchmark_name = f"{protocol}.{subset.value}" + if num_speakers == NumSpeakers.ORACLE: + benchmark_name += ".OracleNumSpeakers" + + with open(into / f"{benchmark_name}.rttm", "w") as rttm: for file in getattr(loaded_protocol, subset.value)(): - prediction: Annotation = pretrained_pipeline(file) + prediction: Annotation = pretrained_pipeline( + file, **file.get("pipeline_kwargs", {}) + ) prediction.write_rttm(rttm) rttm.flush() @@ -211,7 +440,7 @@ def benchmark( if metric is None: return - with open(into / f"{protocol}.{subset.value}.txt", "w") as txt: + with open(into / f"{benchmark_name}.txt", "w") as txt: txt.write(str(metric)) print(str(metric)) diff --git a/uv.lock b/uv.lock index 5ce120a1..8de31245 100644 --- a/uv.lock +++ b/uv.lock @@ -2002,7 +2002,7 @@ wheels = [ [[package]] name = "pyannote-audio" -version = "3.3.3.dev52+ge662b9c2.d20250212" +version = "3.3.3.dev59+g45d30114" source = { editable = "." } dependencies = [ { name = "asteroid-filterbanks" }, @@ -2063,7 +2063,7 @@ requires-dist = [ { name = "pyannote-core", specifier = ">=6.0.0rc1" }, { name = "pyannote-database", specifier = ">=6.0.0rc1" }, { name = "pyannote-metrics", specifier = ">=4.0.0rc1" }, - { name = "pyannote-pipeline", specifier = ">=4.0.0rc1" }, + { name = "pyannote-pipeline", specifier = ">=4.0.0rc2" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytorch-metric-learning", specifier = ">=2.8.1" }, { name = "rich", specifier = ">=13.9.4" }, @@ -2130,7 +2130,7 @@ wheels = [ [[package]] name = "pyannote-pipeline" -version = "4.0.0rc1" +version = "4.0.0rc2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, @@ -2140,9 +2140,9 @@ dependencies = [ { name = "pyyaml" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f1/33/e24b58807851f6ce42de0929470b354dd71983590afb4492634a34440b97/pyannote_pipeline-4.0.0rc1.tar.gz", hash = "sha256:8b1f12b3f122f4608d311ba249bab4b04020f3ee1db68f8317452a1103451100", size = 348744 } +sdist = { url = "https://files.pythonhosted.org/packages/59/5f/c3514701caef9db4b2b30c023a4f7341c71d15af1ecf5574807415650072/pyannote_pipeline-4.0.0rc2.tar.gz", hash = "sha256:ee140379669e5ade9033f3fdea0c3d7fbe865a41bfc1a17a4f1cf47102eb7c30", size = 348886 } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/2e/d3b87cabb483e2c14b10ab799f443397d23554476284703e1eb1edb35f0f/pyannote_pipeline-4.0.0rc1-py3-none-any.whl", hash = "sha256:b8ad4fe06c96787e86d8f53ec26fadda54700c18b77656d7008497124fcb7cb8", size = 22134 }, + { url = "https://files.pythonhosted.org/packages/8c/c8/86d75670b7deab167989fca5dbb050648dd0349c6a9ddd5e97d0bc07286f/pyannote_pipeline-4.0.0rc2-py3-none-any.whl", hash = "sha256:2307b5ba2c2233f61e998a324db06e62f1c4c2517df28140cff591a894ab0c93", size = 22254 }, ] [[package]]