BREAKING: fix cache_dir support, rename use_auth_token to token, and other changes

This commit is contained in:
benniekiss authored and GitHub committed 2025-01-20 22:01:06 +01:00
1 parent ea1ad3429b
commit 2dba593bc5
21 files changed
+233 -472

No files matched your search

+4 -1
View File
@@ -10,7 +10,7 @@ Models can now be stored alongside their pipelines in the same repository, strea
- accept `pyannote/speaker-diarization-x.x` pipeline user agreement
- ~~accept `pyannote/segmentation-3.0` model user agreement~~
- ~~accept `pyannote/wespeaker-voxceleb-resnet34-LM` model user agreement~~
- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", use_auth_token=True)`
- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=True)`
#### Improve speech separation quality
@@ -18,6 +18,9 @@ Clipping and speaker/source alignment issues in speech separation pipeline have
### Breaking changes
- BREAKING(hub): rename `use_auth_token` to `token`
- BREAKING(cache): rely on `huggingface_hub` caching directory (`PYANNOTE_CACHE` is no longer used)
- BREAKING(inference): `Inference` now only supports already instantiated models
- BREAKING(task): drop support for `multilabel` training in `SpeakerDiarization` task
- BREAKING(task): drop support for `warm_up` option in `SpeakerDiarization` task
- BREAKING(task): drop support for `weigh_by_cardinality` option in `SpeakerDiarization` task
+1 -1
View File
@@ -20,7 +20,7 @@ Consider switching to [pyannoteAI](https://www.pyannote.ai) for better and faste
from pyannote.audio import Pipeline
pipeline = Pipeline.from_pretrained(
"pyannote/speaker-diarization-3.1",
use_auth_token="HUGGINGFACE_ACCESS_TOKEN_GOES_HERE")
token="HUGGINGFACE_ACCESS_TOKEN_GOES_HERE")
# send pipeline to GPU (when available)
import torch
+1 -1
View File
@@ -137,7 +137,7 @@ class Calibration(IsotonicRegression):
if os.path.isfile(checkpoint):
return cls.from_file(checkpoint)
path = download_from_hf_hub(
_, _, path = download_from_hf_hub(
checkpoint,
AssetFileName.Calibration,
subfolder=subfolder,
+3 -18
View File
@@ -21,7 +21,6 @@
# SOFTWARE.
import warnings
from pathlib import Path
from typing import Callable, List, Optional, Text, Tuple, Union
import numpy as np
@@ -74,15 +73,11 @@ class Inference(BaseInference):
device : torch.device, optional
Device used for inference. Defaults to `model.device`.
In case `device` and `model.device` are different, model is sent to device.
use_auth_token : str, optional
When loading a private huggingface.co model, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
"""
def __init__(
self,
model: Union[Model, Text, Path],
model: Model,
window: Text = "sliding",
duration: Optional[float] = None,
step: Optional[float] = None,
@@ -91,20 +86,10 @@ class Inference(BaseInference):
skip_conversion: bool = False,
device: Optional[torch.device] = None,
batch_size: int = 32,
use_auth_token: Union[Text, None] = None,
):
# ~~~~ model ~~~~~
self.model = (
model
if isinstance(model, Model)
else Model.from_pretrained(
model,
map_location=device,
strict=False,
use_auth_token=use_auth_token,
)
)
self.model = model
if device is None:
device = self.model.device
@@ -616,7 +601,7 @@ class Inference(BaseInference):
# mask ~ (num_frames_per_chunk, num_classes)-shaped np.ndarray
mask = 1 - np.isnan(score)
np.nan_to_num(score, copy=False, nan=0.0)
start_frame = frames.closest_frame(chunk.start + 0.5 * frames.duration)
aggregated_output[start_frame : start_frame + num_frames_per_chunk] += (
+20 -56
View File
@@ -34,8 +34,6 @@ import pytorch_lightning as pl
import torch
import torch.nn as nn
import torch.optim
from huggingface_hub import hf_hub_download
from huggingface_hub.utils import RepositoryNotFoundError
from lightning_fabric.utilities.cloud_io import _load as pl_load
from pyannote.core import SlidingWindow
from pytorch_lightning.utilities.model_summary.model_summary import ModelSummary
@@ -49,14 +47,10 @@ from pyannote.audio.core.task import (
Task,
UnknownSpecificationsError,
)
from pyannote.audio.utils.hf_hub import AssetFileName, download_from_hf_hub
from pyannote.audio.utils.multi_task import map_with_specifications
from pyannote.audio.utils.version import check_version
CACHE_DIR = os.getenv(
"PYANNOTE_CACHE",
os.path.expanduser("~/.cache/torch/pyannote"),
)
# NOTE: needed to backward compatibility to load models trained before pyannote.audio 3.x
class Introspection:
@@ -83,8 +77,6 @@ class Model(pl.LightningModule):
Task addressed by the model.
"""
MODEL_CHECKPOINT = "pytorch_model.bin"
def __init__(
self,
sample_rate: int = 16000,
@@ -531,10 +523,10 @@ class Model(pl.LightningModule):
map_location=None,
strict: bool = True,
subfolder: Optional[str] = None,
use_auth_token: Union[Text, None] = None, # todo: deprecate in favor of token
cache_dir: Union[Path, Text] = CACHE_DIR,
token: Union[str, bool, None] = None,
cache_dir: Union[Path, str, None] = None,
**kwargs,
) -> "Model":
) -> Optional["Model"]:
"""Load pretrained model
Parameters
@@ -543,7 +535,7 @@ class Model(pl.LightningModule):
Model checkpoint, provided as one of the following:
* path to a local `pytorch_model.bin` model checkpoint
* path to a local directory containing such a file
* identifier of a model on huggingface.co model hub
* identifier of a model on Huggingface hub
map_location: optional
Same role as in torch.load().
Defaults to `lambda storage, loc: storage`.
@@ -552,13 +544,10 @@ class Model(pl.LightningModule):
the keys returned by this module’s state dict. Defaults to True.
subfolder : str, optional
Folder inside the hf.co model repo.
use_auth_token : str, optional
When loading a private hf.co model, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to model cache directory. Defaults to content of PYANNOTE_CACHE
environment variable, or "~/.cache/torch/pyannote" when unset.
Path to the folder where cached files are stored.
kwargs: optional
Any extra keyword args needed to init the model.
Can also be used to override saved hyperparameter values.
@@ -570,18 +559,18 @@ class Model(pl.LightningModule):
See also
--------
torch.load
"""
`huggingface_hub.hf_hub_download`
"""
# if checkpoint is a directory, look for the model checkpoint
# inside this directory (or inside a subfolder if specified)
if os.path.isdir(checkpoint):
if subfolder:
path_to_model_checkpoint = (
Path(checkpoint) / subfolder / cls.MODEL_CHECKPOINT
Path(checkpoint) / subfolder / AssetFileName.Model.value
)
else:
path_to_model_checkpoint = Path(checkpoint) / cls.MODEL_CHECKPOINT
path_to_model_checkpoint = Path(checkpoint) / AssetFileName.Model.value
# if checkpoint is a file, use it as is
elif os.path.isfile(checkpoint):
@@ -589,39 +578,14 @@ class Model(pl.LightningModule):
# otherwise, assume that the checkpoint is hosted on HF model hub
else:
if "@" in checkpoint:
model_id = checkpoint.split("@")[0]
revision = checkpoint.split("@")[1]
else:
model_id = checkpoint
revision = None
try:
path_to_model_checkpoint = hf_hub_download(
model_id,
cls.MODEL_CHECKPOINT,
subfolder=subfolder,
repo_type="model",
revision=revision,
library_name="pyannote",
library_version=__version__,
cache_dir=cache_dir,
token=use_auth_token,
)
except RepositoryNotFoundError:
print(
f"""
Could not download '{model_id}' model.
It might be because the model is private or gated so make
sure to authenticate. Visit https://hf.co/settings/tokens to
create your access token and retry with:
>>> Model.from_pretrained('{model_id}',
... use_auth_token=YOUR_AUTH_TOKEN)
If this still does not work, it might be because the model is gated:
visit https://hf.co/{model_id} to accept the user conditions."""
)
_, _, path_to_model_checkpoint = download_from_hf_hub(
str(checkpoint),
AssetFileName.Model,
subfolder=subfolder,
cache_dir=cache_dir,
token=token,
)
if path_to_model_checkpoint is None:
return None
if map_location is None:
+43 -58
View File
@@ -31,8 +31,6 @@ from typing import Callable, Dict, List, Optional, Text, Union
import torch
import torch.nn as nn
import yaml
from huggingface_hub import hf_hub_download
from huggingface_hub.utils import RepositoryNotFoundError
from pyannote.core.utils.helper import get_class_by_name
from pyannote.database import FileFinder, ProtocolFile
from pyannote.pipeline import Pipeline as _Pipeline
@@ -40,7 +38,8 @@ from pyannote.pipeline import Pipeline as _Pipeline
from pyannote.audio import Audio, __version__
from pyannote.audio.core.inference import BaseInference
from pyannote.audio.core.io import AudioFile
from pyannote.audio.core.model import CACHE_DIR, Model
from pyannote.audio.core.model import Model
from pyannote.audio.utils.hf_hub import AssetFileName, download_from_hf_hub
from pyannote.audio.utils.reproducibility import fix_reproducibility
from pyannote.audio.utils.version import check_version
@@ -48,22 +47,28 @@ from pyannote.audio.utils.version import check_version
def expand_subfolders(
config,
model_id: Optional[Text] = None,
use_auth_token: Optional[Text] = None,
revision: Optional[Text] = None,
cache_dir: Union[Path, str, None] = None,
token: Optional[Text] = None,
) -> None:
"""Expand $model subfolders in config
Processes `config` dictionary recursively and replaces "$model/{subfolder}"
values with {"checkpoint": model_id,
"subfolder": {subfolder},
"use_auth_token": use_auth_token}
"token": token}
Parameters
----------
config : dict
model_id : str, optional
Model identifier when loading from the huggingface.co model hub.
use_auth_token : str, optional
Token used for loading from the root folder.
revision : str, optional
Revision when loading from the huggingface.co model hub.
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
"""
if isinstance(config, dict):
@@ -72,11 +77,15 @@ def expand_subfolders(
subfolder = "/".join(value.split("/")[1:])
config[key] = {
"checkpoint": model_id,
"revision": revision,
"subfolder": subfolder,
"use_auth_token": use_auth_token,
"token": token,
"cache_dir": cache_dir,
}
else:
expand_subfolders(value, model_id, use_auth_token=use_auth_token)
expand_subfolders(
value, model_id, revision=revision, token=token, cache_dir=cache_dir
)
elif isinstance(config, list):
for idx, value in enumerate(config):
@@ -84,24 +93,26 @@ def expand_subfolders(
subfolder = "/".join(value.split("/")[1:])
config[idx] = {
"checkpoint": model_id,
"revision": revision,
"subfolder": subfolder,
"use_auth_token": use_auth_token,
"token": token,
"cache_dir": cache_dir,
}
else:
expand_subfolders(value, model_id, use_auth_token=use_auth_token)
expand_subfolders(
value, model_id, revision=revision, token=token, cache_dir=cache_dir
)
class Pipeline(_Pipeline):
PIPELINE_CHECKPOINT = "config.yaml"
@classmethod
def from_pretrained(
cls,
checkpoint: Union[Text, Path],
hparams_file: Union[Text, Path] = None,
use_auth_token: Union[Text, None] = None, # todo: deprecate in favor of token
cache_dir: Union[Path, Text] = CACHE_DIR,
) -> "Pipeline":
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
) -> Optional["Pipeline"]:
"""Load pretrained pipeline
Parameters
@@ -112,20 +123,17 @@ class Pipeline(_Pipeline):
* path to a local directory containing such a file
* identifier of a pipeline on huggingface.co model hub
hparams_file: Path or str, optional
use_auth_token : str, optional
When loading a private huggingface.co pipeline, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to model cache directory. Defaults to content of PYANNOTE_CACHE
environment variable, or "~/.cache/torch/pyannote" when unset.
Path to the folder where cached files are stored.
"""
# if checkpoint is a directory, look for the pipeline checkpoint
# inside this directory
if os.path.isdir(checkpoint):
model_id = Path(checkpoint)
config_yml = model_id / cls.PIPELINE_CHECKPOINT
config_yml = model_id / AssetFileName.Pipeline.value
# if checkpoint is a file, assume it is the pipeline checkpoint
elif os.path.isfile(checkpoint):
@@ -134,45 +142,21 @@ class Pipeline(_Pipeline):
# otherwise, assume that the checkpoint is hosted on HF model hub
else:
if "@" in checkpoint:
model_id = checkpoint.split("@")[0]
revision = checkpoint.split("@")[1]
else:
model_id = checkpoint
revision = None
try:
config_yml = hf_hub_download(
model_id,
cls.PIPELINE_CHECKPOINT,
repo_type="model",
revision=revision,
library_name="pyannote",
library_version=__version__,
cache_dir=cache_dir,
token=use_auth_token,
)
except RepositoryNotFoundError:
print(
f"""
Could not download '{model_id}' pipeline.
It might be because the pipeline is private or gated so make
sure to authenticate. Visit https://hf.co/settings/tokens to
create your access token and retry with:
>>> Pipeline.from_pretrained('{model_id}',
... use_auth_token=YOUR_AUTH_TOKEN)
If this still does not work, it might be because the pipeline is gated:
visit https://hf.co/{model_id} to accept the user conditions."""
)
model_id, revision, config_yml = download_from_hf_hub(
str(checkpoint),
AssetFileName.Pipeline,
cache_dir=cache_dir,
token=token,
)
if config_yml is None:
return None
with open(config_yml, "r") as fp:
config = yaml.load(fp, Loader=yaml.SafeLoader)
expand_subfolders(config, model_id, use_auth_token=use_auth_token)
expand_subfolders(
config, model_id, revision=revision, token=token, cache_dir=cache_dir
)
if "version" in config:
check_version(
@@ -185,7 +169,8 @@ visit https://hf.co/{model_id} to accept the user conditions."""
pipeline_name, default_module_name="pyannote.pipeline.blocks"
)
params = config["pipeline"].get("params", {})
params.setdefault("use_auth_token", use_auth_token)
params.setdefault("token", token)
params.setdefault("cache_dir", cache_dir)
pipeline = Klass(**params)
# freeze parameters
+8 -6
View File
@@ -25,6 +25,7 @@
# Hervé BREDIN - http://herve.niderb.fr
from functools import partial
from pathlib import Path
from typing import Callable, Optional, Text, Union
from pyannote.core import Annotation, SlidingWindowFeature
@@ -53,10 +54,10 @@ class MultiLabelSegmentation(Pipeline):
Defaults to optimizing identification error rate.
share_min_duration : bool, optional
If True, `min_duration_on` and `min_duration_off` are shared among labels.
use_auth_token : str, optional
When loading private huggingface.co models, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
inference_kwargs : dict, optional
Keywords arguments passed to Inference.
@@ -78,7 +79,8 @@ class MultiLabelSegmentation(Pipeline):
segmentation: Optional[PipelineModel] = None,
fscore: bool = False,
share_min_duration: bool = False,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
**inference_kwargs,
):
@@ -94,7 +96,7 @@ class MultiLabelSegmentation(Pipeline):
self.share_min_duration = share_min_duration
# load model
model = get_model(segmentation, use_auth_token=use_auth_token)
model = get_model(segmentation, token=token, cache_dir=cache_dir)
self._classes = model.specifications.classes
self._segmentation = Inference(model, **inference_kwargs)
@@ -23,6 +23,7 @@
"""Overlapped speech detection pipelines"""
from functools import partial
from pathlib import Path
from typing import Callable, Optional, Text, Union
import numpy as np
@@ -96,10 +97,10 @@ class OverlappedSpeechDetection(Pipeline):
recall : float, optional
Optimize precision at target recall
Defaults to optimize precision/recall fscore
use_auth_token : str, optional
When loading private huggingface.co models, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
inference_kwargs : dict, optional
Keywords arguments passed to Inference.
@@ -118,7 +119,8 @@ class OverlappedSpeechDetection(Pipeline):
segmentation: PipelineModel = "pyannote/segmentation",
precision: Optional[float] = None,
recall: Optional[float] = None,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
**inference_kwargs,
):
super().__init__()
@@ -126,7 +128,7 @@ class OverlappedSpeechDetection(Pipeline):
self.segmentation = segmentation
# load model
model = get_model(segmentation, use_auth_token=use_auth_token)
model = get_model(segmentation, token=token, cache_dir=cache_dir)
if model.dimension > 1:
inference_kwargs["pre_aggregation_hook"] = lambda scores: np.partition(
+8 -6
View File
@@ -23,6 +23,7 @@
"""Resegmentation pipeline"""
from functools import partial
from pathlib import Path
from typing import Callable, Optional, Text, Union
import numpy as np
@@ -67,10 +68,10 @@ class Resegmentation(SpeakerDiarizationMixin, Pipeline):
Optimize for a variant of diarization error rate.
Defaults to {"collar": 0.0, "skip_overlap": False}. This is used in `get_metric`
when instantiating the metric: GreedyDiarizationErrorRate(**der_variant).
use_auth_token : str, optional
When loading private huggingface.co models, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
Hyper-parameters
----------------
@@ -87,14 +88,15 @@ class Resegmentation(SpeakerDiarizationMixin, Pipeline):
segmentation: PipelineModel = "pyannote/segmentation",
diarization: Text = "diarization",
der_variant: Optional[dict] = None,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
):
super().__init__()
self.segmentation = segmentation
self.diarization = diarization
model: Model = get_model(segmentation, use_auth_token=use_auth_token)
model: Model = get_model(segmentation, token=token, cache_dir=cache_dir)
self._segmentation = Inference(model)
self._audio = model.audio
@@ -27,6 +27,7 @@ import itertools
import math
import textwrap
import warnings
from pathlib import Path
from typing import Callable, Mapping, Optional, Text, Union
import numpy as np
@@ -85,10 +86,10 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
Optimize for a variant of diarization error rate.
Defaults to {"collar": 0.0, "skip_overlap": False}. This is used in `get_metric`
when instantiating the metric: GreedyDiarizationErrorRate(**der_variant).
use_auth_token : str, optional
When loading private huggingface.co models, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
Usage
-----
@@ -123,12 +124,13 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
embedding_batch_size: int = 1,
segmentation_batch_size: int = 1,
der_variant: Optional[dict] = None,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
):
super().__init__()
self.segmentation_model = segmentation
model: Model = get_model(segmentation, use_auth_token=use_auth_token)
model: Model = get_model(segmentation, token=token, cache_dir=cache_dir)
self.segmentation_step = segmentation_step
@@ -165,7 +167,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
else:
self._embedding = PretrainedSpeakerEmbedding(
self.embedding, use_auth_token=use_auth_token
self.embedding, token=token, cache_dir=cache_dir
)
self._audio = Audio(sample_rate=self._embedding.sample_rate, mono="downmix")
metric = self._embedding.metric
@@ -36,13 +36,10 @@ from torch.nn.utils.rnn import pad_sequence
from pyannote.audio import Inference, Model, Pipeline
from pyannote.audio.core.inference import BaseInference
from pyannote.audio.core.io import AudioFile
from pyannote.audio.core.model import CACHE_DIR
from pyannote.audio.pipelines.utils import PipelineModel, get_model
try:
from speechbrain.inference import (
EncoderClassifier as SpeechBrain_EncoderClassifier,
)
from speechbrain.inference import EncoderClassifier as SpeechBrain_EncoderClassifier
SPEECHBRAIN_IS_AVAILABLE = True
except ImportError:
@@ -211,10 +208,10 @@ class SpeechBrainPretrainedSpeakerEmbedding(BaseInference):
Name of SpeechBrain model
device : torch.device, optional
Device
use_auth_token : str, optional
When loading private huggingface.co models, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
Usage
-----
@@ -235,7 +232,8 @@ class SpeechBrainPretrainedSpeakerEmbedding(BaseInference):
self,
embedding: Text = "speechbrain/spkrec-ecapa-voxceleb",
device: Optional[torch.device] = None,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
):
if not SPEECHBRAIN_IS_AVAILABLE:
raise ImportError(
@@ -251,13 +249,15 @@ class SpeechBrainPretrainedSpeakerEmbedding(BaseInference):
self.embedding = embedding
self.revision = None
self.device = device or torch.device("cpu")
self.use_auth_token = use_auth_token
self.token = token
self.cache_dir = cache_dir
self.classifier_ = SpeechBrain_EncoderClassifier.from_hparams(
source=self.embedding,
savedir=f"{CACHE_DIR}/speechbrain",
savedir=f"{self.cache_dir}/speechbrain",
run_opts={"device": self.device},
use_auth_token=self.use_auth_token,
token=self.token,
huggingface_cache_dir=self.cache_dir,
revision=self.revision,
)
@@ -269,9 +269,10 @@ class SpeechBrainPretrainedSpeakerEmbedding(BaseInference):
self.classifier_ = SpeechBrain_EncoderClassifier.from_hparams(
source=self.embedding,
savedir=f"{CACHE_DIR}/speechbrain",
savedir=f"{self.cache_dir}/speechbrain",
run_opts={"device": device},
use_auth_token=self.use_auth_token,
token=self.token,
huggingface_cache_dir=self.cache_dir,
revision=self.revision,
)
self.device = device
@@ -391,6 +392,10 @@ class ONNXWeSpeakerPretrainedSpeakerEmbedding(BaseInference):
Path to WeSpeaker pretrained speaker embedding
device : torch.device, optional
Device
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
Usage
-----
@@ -411,6 +416,8 @@ class ONNXWeSpeakerPretrainedSpeakerEmbedding(BaseInference):
self,
embedding: Text = "hbredin/wespeaker-voxceleb-resnet34-LM",
device: Optional[torch.device] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
):
if not ONNX_IS_AVAILABLE:
raise ImportError(
@@ -420,10 +427,13 @@ class ONNXWeSpeakerPretrainedSpeakerEmbedding(BaseInference):
super().__init__()
if not Path(embedding).exists():
try:
embedding = hf_hub_download(
repo_id=embedding,
filename="speaker-embedding.onnx",
token=token,
cache_dir=cache_dir,
)
except RepositoryNotFoundError:
raise ValueError(
@@ -618,10 +628,10 @@ class PyannoteAudioPretrainedSpeakerEmbedding(BaseInference):
pyannote.audio model
device : torch.device, optional
Device
use_auth_token : str, optional
When loading private huggingface.co models, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
Usage
-----
@@ -642,13 +652,14 @@ class PyannoteAudioPretrainedSpeakerEmbedding(BaseInference):
self,
embedding: PipelineModel = "pyannote/embedding",
device: Optional[torch.device] = None,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
):
super().__init__()
self.embedding = embedding
self.device = device or torch.device("cpu")
self.model_: Model = get_model(self.embedding, use_auth_token=use_auth_token)
self.model_: Model = get_model(self.embedding, token=token, cache_dir=cache_dir)
self.model_.eval()
self.model_.to(self.device)
@@ -708,7 +719,8 @@ class PyannoteAudioPretrainedSpeakerEmbedding(BaseInference):
def PretrainedSpeakerEmbedding(
embedding: PipelineModel,
device: Optional[torch.device] = None,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
):
"""Pretrained speaker embedding
@@ -719,10 +731,10 @@ def PretrainedSpeakerEmbedding(
or a pyannote.audio model.
device : torch.device, optional
Device
use_auth_token : str, optional
When loading private huggingface.co models, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
Usage
-----
@@ -743,24 +755,26 @@ def PretrainedSpeakerEmbedding(
if isinstance(embedding, str) and "pyannote" in embedding:
return PyannoteAudioPretrainedSpeakerEmbedding(
embedding, device=device, use_auth_token=use_auth_token
embedding, device=device, token=token, cache_dir=cache_dir
)
elif isinstance(embedding, str) and "speechbrain" in embedding:
return SpeechBrainPretrainedSpeakerEmbedding(
embedding, device=device, use_auth_token=use_auth_token
embedding, device=device, token=token, cache_dir=cache_dir
)
elif isinstance(embedding, str) and "nvidia" in embedding:
return NeMoPretrainedSpeakerEmbedding(embedding, device=device)
elif isinstance(embedding, str) and "wespeaker" in embedding:
return ONNXWeSpeakerPretrainedSpeakerEmbedding(embedding, device=device)
return ONNXWeSpeakerPretrainedSpeakerEmbedding(
embedding, device=device, token=token, cache_dir=cache_dir
)
else:
# fallback to pyannote in case we are loading a local model
return PyannoteAudioPretrainedSpeakerEmbedding(
embedding, device=device, use_auth_token=use_auth_token
embedding, device=device, token=token, cache_dir=cache_dir
)
@@ -779,10 +793,10 @@ class SpeakerEmbedding(Pipeline):
Pretrained segmentation (or voice activity detection) model.
See pyannote.audio.pipelines.utils.get_model for supported format.
Defaults to no voice activity detection.
use_auth_token : str, optional
When loading private huggingface.co models, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
Usage
-----
@@ -798,7 +812,8 @@ class SpeakerEmbedding(Pipeline):
self,
embedding: PipelineModel = "pyannote/embedding",
segmentation: Optional[PipelineModel] = None,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
):
super().__init__()
@@ -806,12 +821,12 @@ class SpeakerEmbedding(Pipeline):
self.segmentation = segmentation
self.embedding_model_: Model = get_model(
embedding, use_auth_token=use_auth_token
embedding, token=token, cache_dir=cache_dir
)
if self.segmentation is not None:
segmentation_model: Model = get_model(
self.segmentation, use_auth_token=use_auth_token
self.segmentation, token=token, cache_dir=cache_dir
)
self._segmentation = Inference(
segmentation_model,
@@ -27,6 +27,7 @@ import itertools
import math
import textwrap
import warnings
from pathlib import Path
from typing import Callable, Optional, Text, Tuple, Union
import numpy as np
@@ -86,10 +87,10 @@ class SpeechSeparation(SpeakerDiarizationMixin, Pipeline):
Optimize for a variant of diarization error rate.
Defaults to {"collar": 0.0, "skip_overlap": False}. This is used in `get_metric`
when instantiating the metric: GreedyDiarizationErrorRate(**der_variant).
use_auth_token : str, optional
When loading private huggingface.co models, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
Usage
-----
@@ -133,12 +134,13 @@ class SpeechSeparation(SpeakerDiarizationMixin, Pipeline):
embedding_batch_size: int = 1,
segmentation_batch_size: int = 1,
der_variant: Optional[dict] = None,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
):
super().__init__()
self.segmentation_model = segmentation
model: Model = get_model(segmentation, use_auth_token=use_auth_token)
model: Model = get_model(segmentation, token=token, cache_dir=cache_dir)
self.segmentation_step = segmentation_step
@@ -175,7 +177,7 @@ class SpeechSeparation(SpeakerDiarizationMixin, Pipeline):
else:
self._embedding = PretrainedSpeakerEmbedding(
self.embedding, use_auth_token=use_auth_token
self.embedding, token=token, cache_dir=cache_dir
)
self._audio = Audio(sample_rate=self._embedding.sample_rate, mono="downmix")
metric = self._embedding.metric
@@ -24,12 +24,10 @@ from .diarization import SpeakerDiarizationMixin
from .getter import (
PipelineAugmentation,
PipelineCalibration,
PipelineInference,
PipelineModel,
get_augmentation,
get_calibration,
get_devices,
get_inference,
get_model,
)
from .oracle import oracle_segmentation
@@ -40,8 +38,6 @@ __all__ = [
"get_augmentation",
"PipelineAugmentation",
"get_devices",
"get_inference",
"PipelineInference",
"get_model",
"PipelineModel",
"get_calibration",
+23 -60
View File
@@ -21,21 +21,23 @@
# SOFTWARE.
import itertools
from pathlib import Path
from typing import Dict, Mapping, Optional, Text, Union
import torch
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
from torch_audiomentations.utils.config import from_dict as augmentation_from_dict
from pyannote.audio import Inference, Model
from pyannote.audio.core.calibration import Calibration
from pyannote.audio.core.model import Model
PipelineModel = Union[Model, Text, Mapping]
def get_model(
model: PipelineModel,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
) -> Model:
"""Load pretrained model and set it into `eval` mode.
@@ -46,10 +48,10 @@ def get_model(
When `str`, assumes that this is either the path to a checkpoint or the name of a
pretrained model on Huggingface.co and loads with `Model.from_pretrained(model)`
When `dict`, loads with `Model.from_pretrained(**model)`.
use_auth_token : str, optional
When loading a private or gated huggingface.co pipeline, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by visiting https://hf.co/settings/tokens
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
Returns
-------
@@ -73,12 +75,15 @@ def get_model(
pass
elif isinstance(model, Text):
model = Model.from_pretrained(
model, use_auth_token=use_auth_token, strict=False
_model = Model.from_pretrained(
model, token=token, cache_dir=cache_dir, strict=False
)
if _model:
model = _model
elif isinstance(model, Mapping):
model.setdefault("use_auth_token", use_auth_token)
model.setdefault("token", token)
model.setdefault("cache_dir", cache_dir)
model = Model.from_pretrained(**model)
else:
@@ -91,54 +96,6 @@ def get_model(
return model
PipelineInference = Union[Inference, Model, Text, Mapping]
def get_inference(inference: PipelineInference) -> Inference:
"""Load inference
Parameter
---------
inference : Inference, Model, str, or dict
When `Inference`, returns `inference` as is.
When `Model`, wraps it in `Inference(model)`.
When `str`, assumes that this is either the path to a checkpoint or the name of a
pretrained model on Huggingface.co and loads with `Inference(checkpoint)`.
When `dict`, loads with `Inference(**inference)`.
Returns
-------
inference : Inference
Inference.
Examples
--------
>>> inference = get_inference("hbredin/VoiceActivityDetection-PyanNet-DIHARD")
>>> inference = get_inference("/path/to/checkpoint.ckpt")
>>> inference = get_inference({"model": "hbredin/VoiceActivityDetection-PyanNet-DIHARD",
... "window": "sliding"})
See also
--------
pyannote.audio.core.inference.Inference
"""
if isinstance(inference, Inference):
return inference
if isinstance(inference, (Model, Text)):
return Inference(inference)
if isinstance(inference, Mapping):
return Inference(**inference)
raise TypeError(
f"Unsupported type ({type(inference)}) for loading inference: "
f"expected `Model`, `str` or `dict`."
)
PipelineAugmentation = Union[BaseWaveformTransform, Mapping]
@@ -147,7 +104,8 @@ PipelineCalibration = Union[Calibration, Text, Dict]
def get_calibration(
calibration: PipelineCalibration,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
) -> Optional[Calibration]:
"""Load pretrained calibration
@@ -158,6 +116,10 @@ def get_calibration(
When `str`, assumes that this is either the path to a checkpoint or the name of a
pretrained calibration on Huggingface.co and loads with `Calibration.from_pretrained(calibration)`.
When `dict`, loads with `Calibration.from_pretrained(**calibration)`.
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
Returns
-------
@@ -174,11 +136,12 @@ def get_calibration(
elif isinstance(calibration, Text):
loaded_calibration = Calibration.from_pretrained(
calibration, use_auth_token=use_auth_token
calibration, token=token, cache_dir=cache_dir
)
elif isinstance(calibration, Dict):
calibration.setdefault("use_auth_token", use_auth_token)
calibration.setdefault("token", token)
calibration.setdefault("cache_dir", cache_dir)
loaded_calibration = Calibration.from_pretrained(**calibration)
else:
@@ -22,37 +22,22 @@
"""Voice activity detection pipelines"""
import tempfile
from copy import deepcopy
from functools import partial
from types import MethodType
from pathlib import Path
from typing import Callable, Optional, Text, Union
import numpy as np
from pyannote.core import Annotation, SlidingWindowFeature
from pyannote.database.protocol import SpeakerDiarizationProtocol
from pyannote.metrics.detection import (
DetectionErrorRate,
DetectionPrecisionRecallFMeasure,
)
from pyannote.pipeline.parameter import Categorical, Integer, LogUniform, Uniform
from pytorch_lightning import Trainer
from torch.optim import SGD
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
from pyannote.pipeline.parameter import Uniform
from pyannote.audio import Inference
from pyannote.audio.core.callback import GraduallyUnfreeze
from pyannote.audio.core.io import AudioFile
from pyannote.audio.core.pipeline import Pipeline
from pyannote.audio.pipelines.utils import (
PipelineAugmentation,
PipelineInference,
PipelineModel,
get_augmentation,
get_inference,
get_model,
)
from pyannote.audio.tasks import VoiceActivityDetection as VoiceActivityDetectionTask
from pyannote.audio.pipelines.utils import PipelineModel, get_model
from pyannote.audio.utils.signal import Binarize
@@ -90,10 +75,10 @@ class VoiceActivityDetection(Pipeline):
fscore : bool, optional
Optimize (precision/recall) fscore. Defaults to optimizing detection
error rate.
use_auth_token : str, optional
When loading private huggingface.co models, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
inference_kwargs : dict, optional
Keywords arguments passed to Inference.
@@ -111,7 +96,8 @@ class VoiceActivityDetection(Pipeline):
self,
segmentation: PipelineModel = "pyannote/segmentation",
fscore: bool = False,
use_auth_token: Union[Text, None] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
**inference_kwargs,
):
super().__init__()
@@ -120,7 +106,7 @@ class VoiceActivityDetection(Pipeline):
self.fscore = fscore
# load model and send it to GPU (when available and not already on GPU)
model = get_model(segmentation, use_auth_token=use_auth_token)
model = get_model(segmentation, token=token, cache_dir=cache_dir)
inference_kwargs["pre_aggregation_hook"] = lambda scores: np.max(
scores, axis=-1, keepdims=True
@@ -230,158 +216,3 @@ class VoiceActivityDetection(Pipeline):
if self.fscore:
return "maximize"
return "minimize"
class AdaptiveVoiceActivityDetection(Pipeline):
"""Adaptive voice activity detection pipeline
Let M be a pretrained voice activity detection model.
For each file f, this pipeline starts by applying the model to obtain a first set of
speech/non-speech labels.
Those (automatic, possibly erroneous) labels are then used to fine-tune M on the very
same file f into a M_f model, in a self-supervised manner.
Finally, the fine-tuned model M_f is applied to file f to obtain the final (and
hopefully better) speech/non-speech labels.
During fine-tuning, frames where the pretrained model M is very confident are weighted
more than those with lower confidence: the intuition is that the model will use these
high confidence regions to adapt to recording conditions (e.g. background noise) and
hence will eventually be better on the parts of f where it was initially not quite
confident.
Conversely, to avoid overfitting too much to those high confidence regions, we use
data augmentation and freeze all but the final few layers of the pretrained model M.
Parameters
----------
segmentation : Model, str, or dict, optional
Pretrained segmentation model.
Defaults to "hbredin/VoiceActivityDetection-PyanNet-DIHARD".
augmentation : BaseWaveformTransform, or dict, optional
torch_audiomentations waveform transform, used during fine-tuning.
Defaults to no augmentation.
fscore : bool, optional
Optimize (precision/recall) fscore.
Defaults to optimizing detection error rate.
Hyper-parameters
----------------
num_epochs : int
Number of epochs (where one epoch = going through the file once).
batch_size : int
Batch size.
learning_rate : float
Learning rate.
See also
--------
pyannote.audio.pipelines.utils.get_inference
"""
def __init__(
self,
segmentation: PipelineInference = "hbredin/VoiceActivityDetection-PyanNet-DIHARD",
augmentation: Optional[PipelineAugmentation] = None,
fscore: bool = False,
):
super().__init__()
# pretrained segmentation model
self.inference: Inference = get_inference(segmentation)
self.augmentation: BaseWaveformTransform = get_augmentation(augmentation)
self.fscore = fscore
self.num_epochs = Integer(0, 10)
self.batch_size = Categorical([1, 2, 4, 8, 16, 32])
self.learning_rate = LogUniform(1e-6, 1)
def apply(self, file: AudioFile) -> Annotation:
# create a copy of file
file = dict(file)
# get segmentation scores from pretrained segmentation model
file["seg"] = self.inference(file)
# infer voice activity detection scores
file["vad"] = np.max(file["seg"], axis=1, keepdims=True)
# apply voice activity detection pipeline with default parameters
vad_pipeline = VoiceActivityDetection("vad").instantiate(
{
"onset": 0.5,
"offset": 0.5,
"min_duration_on": 0.0,
"min_duration_off": 0.0,
}
)
file["annotation"] = vad_pipeline(file)
# do not fine tune the model if num_epochs is zero
if self.num_epochs == 0:
return file["annotation"]
# infer model confidence from segmentation scores
# TODO: scale confidence differently (e.g. via an additional binarisation threshold hyper-parameter)
file["confidence"] = np.min(
np.abs((file["seg"] - 0.5) / 0.5), axis=1, keepdims=True
)
# create a dummy train-only protocol where `file` is the only training file
class DummyProtocol(SpeakerDiarizationProtocol):
name = "DummyProtocol"
def train_iter(self):
yield file
vad_task = VoiceActivityDetectionTask(
DummyProtocol(),
duration=self.inference.duration,
weight="confidence",
batch_size=self.batch_size,
augmentation=self.augmentation,
)
vad_model = deepcopy(self.inference.model)
vad_model.task = vad_task
def configure_optimizers(model):
return SGD(model.parameters(), lr=self.learning_rate)
vad_model.configure_optimizers = MethodType(configure_optimizers, vad_model)
with tempfile.TemporaryDirectory() as default_root_dir:
trainer = Trainer(
max_epochs=self.num_epochs,
accelerator="gpu",
devices=1,
callbacks=[GraduallyUnfreeze(epochs_per_stage=self.num_epochs + 1)],
enable_checkpointing=False,
default_root_dir=default_root_dir,
)
trainer.fit(vad_model)
inference = Inference(
vad_model,
device=self.inference.device,
batch_size=self.inference.batch_size,
)
file["vad"] = inference(file)
return vad_pipeline(file)
def get_metric(self) -> Union[DetectionErrorRate, DetectionPrecisionRecallFMeasure]:
"""Return new instance of detection metric"""
if self.fscore:
return DetectionPrecisionRecallFMeasure(collar=0.0, skip_overlap=False)
return DetectionErrorRate(collar=0.0, skip_overlap=False)
def get_direction(self):
if self.fscore:
return "maximize"
return "minimize"
+27 -19
View File
@@ -22,11 +22,13 @@
from enum import Enum
from pathlib import Path
from typing import Optional, Union
from typing import Optional, Tuple, Union
from huggingface_hub import hf_hub_download
from huggingface_hub.utils import HfHubHTTPError
from pyannote.audio import __version__
# Correspondence between asset_file type
# and their filename on Huggingface
@@ -42,7 +44,7 @@ def download_from_hf_hub(
subfolder: Optional[str] = None,
cache_dir: Optional[Union[str, Path]] = None,
token: Union[bool, str, None] = None,
) -> Optional[str]:
) -> Tuple[str, Optional[str], Optional[str]]:
"""Download file from Huggingface Hub
Parameters
@@ -52,14 +54,15 @@ def download_from_hf_hub(
asset_file : AssetFileName
Type of asset file to download.
subfolder : str, optional
Folder inside the hf.co model repo.
token : str, optional
When loading a private hf.co model, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
token that can be obtained by running `huggingface-cli login`
Folder inside the model repo.
token : str or bool, optional
Token to be used for the download.
cache_dir: Path or str, optional
Path to model cache directory. Defaults to content of PYANNOTE_CACHE
environment variable, or "~/.cache/torch/pyannote" when unset.
Path to the folder where cached files are stored.
See also
--------
`huggingface_hub.hf_hub_download`
"""
if "@" in checkpoint:
@@ -68,15 +71,20 @@ def download_from_hf_hub(
model_id, revision = checkpoint, None
try:
return hf_hub_download(
return (
model_id,
asset_file.value,
subfolder=subfolder,
repo_type="model",
revision=revision,
library_name="pyannote",
cache_dir=cache_dir,
token=token,
revision,
hf_hub_download(
model_id,
asset_file.value,
subfolder=subfolder,
repo_type="model",
revision=revision,
library_name="pyannote",
library_version=__version__,
cache_dir=cache_dir,
token=token,
),
)
except HfHubHTTPError:
print(
@@ -87,7 +95,7 @@ It might be because the repository is private or gated:
* visit https://hf.co/{model_id} to accept user conditions
* visit https://hf.co/settings/tokens to create an authentication token
* load the {asset_file.name.lower()} with the `token` argument:
>>> {asset_file.name}.from_pretrained('{model_id}', use_auth_token=...)
>>> {asset_file.name}.from_pretrained('{model_id}', token=...)
"""
)
return
raise
+2 -1
View File
@@ -13,7 +13,8 @@ HF_SAMPLE_MODEL_ID = "pyannote/ci-segmentation"
def test_hf_download_inference():
inference = Inference(HF_SAMPLE_MODEL_ID, device="cpu")
model = Model.from_pretrained(HF_SAMPLE_MODEL_ID)
inference = Inference(model, device="cpu")
assert isinstance(inference, Inference)
+2 -2
View File
@@ -197,7 +197,7 @@
"outputs": [],
"source": [
"from pyannote.audio import Pipeline\n",
"pretrained_pipeline = Pipeline.from_pretrained(\"pyannote/speaker-diarization\", use_auth_token=True) "
"pretrained_pipeline = Pipeline.from_pretrained(\"pyannote/speaker-diarization\", token=True) "
]
},
{
@@ -312,7 +312,7 @@
"outputs": [],
"source": [
"from pyannote.audio import Model\n",
"model = Model.from_pretrained(\"pyannote/segmentation\", use_auth_token=True)"
"model = Model.from_pretrained(\"pyannote/segmentation\", token=True)"
]
},
{
@@ -333,7 +333,7 @@
"source": [
"from pyannote.audio import Pipeline\n",
"pipeline = Pipeline.from_pretrained(\n",
" \"pyannote/speech-separation-ami-1.0\", use_auth_token=True)\n",
" \"pyannote/speech-separation-ami-1.0\", token=True)\n",
"\n",
"# you can safely ignore that various warnings below."
]
@@ -961,7 +961,7 @@
"from pyannote.audio.core.model import Model\n",
"\n",
"# apply totatonet on the first five seconds\n",
"totatonet = Model.from_pretrained(\"pyannote/separation-ami-1.0\", use_auth_token=True)\n",
"totatonet = Model.from_pretrained(\"pyannote/separation-ami-1.0\", token=True)\n",
"totatonet.to(device)\n",
"\n",
"cropped_mixture = mixture[:, :sample_rate * 5].unsqueeze(0)\n",
+1 -1
View File
@@ -413,7 +413,7 @@
"source": [
"# load pretrained pipeline\n",
"from pyannote.audio import Pipeline\n",
"pipeline = Pipeline.from_pretrained('pyannote/speaker-diarization-3.1', use_auth_token=True)\n",
"pipeline = Pipeline.from_pretrained('pyannote/speaker-diarization-3.1', token=True)\n",
"\n",
"# send pipeline to GPU (when available)\n",
"import torch\n",
+1 -1
View File
@@ -258,7 +258,7 @@
"source": [
"from pyannote.audio.core.model import Model\n",
"\n",
"pretrained_model = Model.from_pretrained(\"pyannote/segmentation-3.0\", use_auth_token=True)\n",
"pretrained_model = Model.from_pretrained(\"pyannote/segmentation-3.0\", token=True)\n",
"# we assign the OSD task to the model\n",
"pretrained_model.task = osd"
]