mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
BREAKING: fix cache_dir support, rename use_auth_token to token, and other changes
This commit is contained in:
1 parent
ea1ad3429b
commit
2dba593bc5
21 files changed
+233
-472
No files matched your search
+4
-1
@@ -10,7 +10,7 @@ Models can now be stored alongside their pipelines in the same repository, strea
|
||||
- accept `pyannote/speaker-diarization-x.x` pipeline user agreement
|
||||
- ~~accept `pyannote/segmentation-3.0` model user agreement~~
|
||||
- ~~accept `pyannote/wespeaker-voxceleb-resnet34-LM` model user agreement~~
|
||||
- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", use_auth_token=True)`
|
||||
- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=True)`
|
||||
|
||||
#### Improve speech separation quality
|
||||
|
||||
@@ -18,6 +18,9 @@ Clipping and speaker/source alignment issues in speech separation pipeline have
|
||||
|
||||
### Breaking changes
|
||||
|
||||
- BREAKING(hub): rename `use_auth_token` to `token`
|
||||
- BREAKING(cache): rely on `huggingface_hub` caching directory (`PYANNOTE_CACHE` is no longer used)
|
||||
- BREAKING(inference): `Inference` now only supports already instantiated models
|
||||
- BREAKING(task): drop support for `multilabel` training in `SpeakerDiarization` task
|
||||
- BREAKING(task): drop support for `warm_up` option in `SpeakerDiarization` task
|
||||
- BREAKING(task): drop support for `weigh_by_cardinality` option in `SpeakerDiarization` task
|
||||
|
||||
@@ -20,7 +20,7 @@ Consider switching to [pyannoteAI](https://www.pyannote.ai) for better and faste
|
||||
from pyannote.audio import Pipeline
|
||||
pipeline = Pipeline.from_pretrained(
|
||||
"pyannote/speaker-diarization-3.1",
|
||||
use_auth_token="HUGGINGFACE_ACCESS_TOKEN_GOES_HERE")
|
||||
token="HUGGINGFACE_ACCESS_TOKEN_GOES_HERE")
|
||||
|
||||
# send pipeline to GPU (when available)
|
||||
import torch
|
||||
|
||||
@@ -137,7 +137,7 @@ class Calibration(IsotonicRegression):
|
||||
if os.path.isfile(checkpoint):
|
||||
return cls.from_file(checkpoint)
|
||||
|
||||
path = download_from_hf_hub(
|
||||
_, _, path = download_from_hf_hub(
|
||||
checkpoint,
|
||||
AssetFileName.Calibration,
|
||||
subfolder=subfolder,
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
# SOFTWARE.
|
||||
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Callable, List, Optional, Text, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
@@ -74,15 +73,11 @@ class Inference(BaseInference):
|
||||
device : torch.device, optional
|
||||
Device used for inference. Defaults to `model.device`.
|
||||
In case `device` and `model.device` are different, model is sent to device.
|
||||
use_auth_token : str, optional
|
||||
When loading a private huggingface.co model, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: Union[Model, Text, Path],
|
||||
model: Model,
|
||||
window: Text = "sliding",
|
||||
duration: Optional[float] = None,
|
||||
step: Optional[float] = None,
|
||||
@@ -91,20 +86,10 @@ class Inference(BaseInference):
|
||||
skip_conversion: bool = False,
|
||||
device: Optional[torch.device] = None,
|
||||
batch_size: int = 32,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
):
|
||||
# ~~~~ model ~~~~~
|
||||
|
||||
self.model = (
|
||||
model
|
||||
if isinstance(model, Model)
|
||||
else Model.from_pretrained(
|
||||
model,
|
||||
map_location=device,
|
||||
strict=False,
|
||||
use_auth_token=use_auth_token,
|
||||
)
|
||||
)
|
||||
self.model = model
|
||||
|
||||
if device is None:
|
||||
device = self.model.device
|
||||
@@ -616,7 +601,7 @@ class Inference(BaseInference):
|
||||
# mask ~ (num_frames_per_chunk, num_classes)-shaped np.ndarray
|
||||
mask = 1 - np.isnan(score)
|
||||
np.nan_to_num(score, copy=False, nan=0.0)
|
||||
|
||||
|
||||
start_frame = frames.closest_frame(chunk.start + 0.5 * frames.duration)
|
||||
|
||||
aggregated_output[start_frame : start_frame + num_frames_per_chunk] += (
|
||||
|
||||
@@ -34,8 +34,6 @@ import pytorch_lightning as pl
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.optim
|
||||
from huggingface_hub import hf_hub_download
|
||||
from huggingface_hub.utils import RepositoryNotFoundError
|
||||
from lightning_fabric.utilities.cloud_io import _load as pl_load
|
||||
from pyannote.core import SlidingWindow
|
||||
from pytorch_lightning.utilities.model_summary.model_summary import ModelSummary
|
||||
@@ -49,14 +47,10 @@ from pyannote.audio.core.task import (
|
||||
Task,
|
||||
UnknownSpecificationsError,
|
||||
)
|
||||
from pyannote.audio.utils.hf_hub import AssetFileName, download_from_hf_hub
|
||||
from pyannote.audio.utils.multi_task import map_with_specifications
|
||||
from pyannote.audio.utils.version import check_version
|
||||
|
||||
CACHE_DIR = os.getenv(
|
||||
"PYANNOTE_CACHE",
|
||||
os.path.expanduser("~/.cache/torch/pyannote"),
|
||||
)
|
||||
|
||||
|
||||
# NOTE: needed to backward compatibility to load models trained before pyannote.audio 3.x
|
||||
class Introspection:
|
||||
@@ -83,8 +77,6 @@ class Model(pl.LightningModule):
|
||||
Task addressed by the model.
|
||||
"""
|
||||
|
||||
MODEL_CHECKPOINT = "pytorch_model.bin"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sample_rate: int = 16000,
|
||||
@@ -531,10 +523,10 @@ class Model(pl.LightningModule):
|
||||
map_location=None,
|
||||
strict: bool = True,
|
||||
subfolder: Optional[str] = None,
|
||||
use_auth_token: Union[Text, None] = None, # todo: deprecate in favor of token
|
||||
cache_dir: Union[Path, Text] = CACHE_DIR,
|
||||
token: Union[str, bool, None] = None,
|
||||
cache_dir: Union[Path, str, None] = None,
|
||||
**kwargs,
|
||||
) -> "Model":
|
||||
) -> Optional["Model"]:
|
||||
"""Load pretrained model
|
||||
|
||||
Parameters
|
||||
@@ -543,7 +535,7 @@ class Model(pl.LightningModule):
|
||||
Model checkpoint, provided as one of the following:
|
||||
* path to a local `pytorch_model.bin` model checkpoint
|
||||
* path to a local directory containing such a file
|
||||
* identifier of a model on huggingface.co model hub
|
||||
* identifier of a model on Huggingface hub
|
||||
map_location: optional
|
||||
Same role as in torch.load().
|
||||
Defaults to `lambda storage, loc: storage`.
|
||||
@@ -552,13 +544,10 @@ class Model(pl.LightningModule):
|
||||
the keys returned by this module’s state dict. Defaults to True.
|
||||
subfolder : str, optional
|
||||
Folder inside the hf.co model repo.
|
||||
use_auth_token : str, optional
|
||||
When loading a private hf.co model, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to model cache directory. Defaults to content of PYANNOTE_CACHE
|
||||
environment variable, or "~/.cache/torch/pyannote" when unset.
|
||||
Path to the folder where cached files are stored.
|
||||
kwargs: optional
|
||||
Any extra keyword args needed to init the model.
|
||||
Can also be used to override saved hyperparameter values.
|
||||
@@ -570,18 +559,18 @@ class Model(pl.LightningModule):
|
||||
|
||||
See also
|
||||
--------
|
||||
torch.load
|
||||
"""
|
||||
`huggingface_hub.hf_hub_download`
|
||||
|
||||
"""
|
||||
# if checkpoint is a directory, look for the model checkpoint
|
||||
# inside this directory (or inside a subfolder if specified)
|
||||
if os.path.isdir(checkpoint):
|
||||
if subfolder:
|
||||
path_to_model_checkpoint = (
|
||||
Path(checkpoint) / subfolder / cls.MODEL_CHECKPOINT
|
||||
Path(checkpoint) / subfolder / AssetFileName.Model.value
|
||||
)
|
||||
else:
|
||||
path_to_model_checkpoint = Path(checkpoint) / cls.MODEL_CHECKPOINT
|
||||
path_to_model_checkpoint = Path(checkpoint) / AssetFileName.Model.value
|
||||
|
||||
# if checkpoint is a file, use it as is
|
||||
elif os.path.isfile(checkpoint):
|
||||
@@ -589,39 +578,14 @@ class Model(pl.LightningModule):
|
||||
|
||||
# otherwise, assume that the checkpoint is hosted on HF model hub
|
||||
else:
|
||||
if "@" in checkpoint:
|
||||
model_id = checkpoint.split("@")[0]
|
||||
revision = checkpoint.split("@")[1]
|
||||
else:
|
||||
model_id = checkpoint
|
||||
revision = None
|
||||
|
||||
try:
|
||||
path_to_model_checkpoint = hf_hub_download(
|
||||
model_id,
|
||||
cls.MODEL_CHECKPOINT,
|
||||
subfolder=subfolder,
|
||||
repo_type="model",
|
||||
revision=revision,
|
||||
library_name="pyannote",
|
||||
library_version=__version__,
|
||||
cache_dir=cache_dir,
|
||||
token=use_auth_token,
|
||||
)
|
||||
except RepositoryNotFoundError:
|
||||
print(
|
||||
f"""
|
||||
Could not download '{model_id}' model.
|
||||
It might be because the model is private or gated so make
|
||||
sure to authenticate. Visit https://hf.co/settings/tokens to
|
||||
create your access token and retry with:
|
||||
|
||||
>>> Model.from_pretrained('{model_id}',
|
||||
... use_auth_token=YOUR_AUTH_TOKEN)
|
||||
|
||||
If this still does not work, it might be because the model is gated:
|
||||
visit https://hf.co/{model_id} to accept the user conditions."""
|
||||
)
|
||||
_, _, path_to_model_checkpoint = download_from_hf_hub(
|
||||
str(checkpoint),
|
||||
AssetFileName.Model,
|
||||
subfolder=subfolder,
|
||||
cache_dir=cache_dir,
|
||||
token=token,
|
||||
)
|
||||
if path_to_model_checkpoint is None:
|
||||
return None
|
||||
|
||||
if map_location is None:
|
||||
|
||||
@@ -31,8 +31,6 @@ from typing import Callable, Dict, List, Optional, Text, Union
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import yaml
|
||||
from huggingface_hub import hf_hub_download
|
||||
from huggingface_hub.utils import RepositoryNotFoundError
|
||||
from pyannote.core.utils.helper import get_class_by_name
|
||||
from pyannote.database import FileFinder, ProtocolFile
|
||||
from pyannote.pipeline import Pipeline as _Pipeline
|
||||
@@ -40,7 +38,8 @@ from pyannote.pipeline import Pipeline as _Pipeline
|
||||
from pyannote.audio import Audio, __version__
|
||||
from pyannote.audio.core.inference import BaseInference
|
||||
from pyannote.audio.core.io import AudioFile
|
||||
from pyannote.audio.core.model import CACHE_DIR, Model
|
||||
from pyannote.audio.core.model import Model
|
||||
from pyannote.audio.utils.hf_hub import AssetFileName, download_from_hf_hub
|
||||
from pyannote.audio.utils.reproducibility import fix_reproducibility
|
||||
from pyannote.audio.utils.version import check_version
|
||||
|
||||
@@ -48,22 +47,28 @@ from pyannote.audio.utils.version import check_version
|
||||
def expand_subfolders(
|
||||
config,
|
||||
model_id: Optional[Text] = None,
|
||||
use_auth_token: Optional[Text] = None,
|
||||
revision: Optional[Text] = None,
|
||||
cache_dir: Union[Path, str, None] = None,
|
||||
token: Optional[Text] = None,
|
||||
) -> None:
|
||||
"""Expand $model subfolders in config
|
||||
|
||||
Processes `config` dictionary recursively and replaces "$model/{subfolder}"
|
||||
values with {"checkpoint": model_id,
|
||||
"subfolder": {subfolder},
|
||||
"use_auth_token": use_auth_token}
|
||||
"token": token}
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config : dict
|
||||
model_id : str, optional
|
||||
Model identifier when loading from the huggingface.co model hub.
|
||||
use_auth_token : str, optional
|
||||
Token used for loading from the root folder.
|
||||
revision : str, optional
|
||||
Revision when loading from the huggingface.co model hub.
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
"""
|
||||
|
||||
if isinstance(config, dict):
|
||||
@@ -72,11 +77,15 @@ def expand_subfolders(
|
||||
subfolder = "/".join(value.split("/")[1:])
|
||||
config[key] = {
|
||||
"checkpoint": model_id,
|
||||
"revision": revision,
|
||||
"subfolder": subfolder,
|
||||
"use_auth_token": use_auth_token,
|
||||
"token": token,
|
||||
"cache_dir": cache_dir,
|
||||
}
|
||||
else:
|
||||
expand_subfolders(value, model_id, use_auth_token=use_auth_token)
|
||||
expand_subfolders(
|
||||
value, model_id, revision=revision, token=token, cache_dir=cache_dir
|
||||
)
|
||||
|
||||
elif isinstance(config, list):
|
||||
for idx, value in enumerate(config):
|
||||
@@ -84,24 +93,26 @@ def expand_subfolders(
|
||||
subfolder = "/".join(value.split("/")[1:])
|
||||
config[idx] = {
|
||||
"checkpoint": model_id,
|
||||
"revision": revision,
|
||||
"subfolder": subfolder,
|
||||
"use_auth_token": use_auth_token,
|
||||
"token": token,
|
||||
"cache_dir": cache_dir,
|
||||
}
|
||||
else:
|
||||
expand_subfolders(value, model_id, use_auth_token=use_auth_token)
|
||||
expand_subfolders(
|
||||
value, model_id, revision=revision, token=token, cache_dir=cache_dir
|
||||
)
|
||||
|
||||
|
||||
class Pipeline(_Pipeline):
|
||||
PIPELINE_CHECKPOINT = "config.yaml"
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(
|
||||
cls,
|
||||
checkpoint: Union[Text, Path],
|
||||
hparams_file: Union[Text, Path] = None,
|
||||
use_auth_token: Union[Text, None] = None, # todo: deprecate in favor of token
|
||||
cache_dir: Union[Path, Text] = CACHE_DIR,
|
||||
) -> "Pipeline":
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
) -> Optional["Pipeline"]:
|
||||
"""Load pretrained pipeline
|
||||
|
||||
Parameters
|
||||
@@ -112,20 +123,17 @@ class Pipeline(_Pipeline):
|
||||
* path to a local directory containing such a file
|
||||
* identifier of a pipeline on huggingface.co model hub
|
||||
hparams_file: Path or str, optional
|
||||
use_auth_token : str, optional
|
||||
When loading a private huggingface.co pipeline, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to model cache directory. Defaults to content of PYANNOTE_CACHE
|
||||
environment variable, or "~/.cache/torch/pyannote" when unset.
|
||||
Path to the folder where cached files are stored.
|
||||
"""
|
||||
|
||||
# if checkpoint is a directory, look for the pipeline checkpoint
|
||||
# inside this directory
|
||||
if os.path.isdir(checkpoint):
|
||||
model_id = Path(checkpoint)
|
||||
config_yml = model_id / cls.PIPELINE_CHECKPOINT
|
||||
config_yml = model_id / AssetFileName.Pipeline.value
|
||||
|
||||
# if checkpoint is a file, assume it is the pipeline checkpoint
|
||||
elif os.path.isfile(checkpoint):
|
||||
@@ -134,45 +142,21 @@ class Pipeline(_Pipeline):
|
||||
|
||||
# otherwise, assume that the checkpoint is hosted on HF model hub
|
||||
else:
|
||||
if "@" in checkpoint:
|
||||
model_id = checkpoint.split("@")[0]
|
||||
revision = checkpoint.split("@")[1]
|
||||
else:
|
||||
model_id = checkpoint
|
||||
revision = None
|
||||
|
||||
try:
|
||||
config_yml = hf_hub_download(
|
||||
model_id,
|
||||
cls.PIPELINE_CHECKPOINT,
|
||||
repo_type="model",
|
||||
revision=revision,
|
||||
library_name="pyannote",
|
||||
library_version=__version__,
|
||||
cache_dir=cache_dir,
|
||||
token=use_auth_token,
|
||||
)
|
||||
|
||||
except RepositoryNotFoundError:
|
||||
print(
|
||||
f"""
|
||||
Could not download '{model_id}' pipeline.
|
||||
It might be because the pipeline is private or gated so make
|
||||
sure to authenticate. Visit https://hf.co/settings/tokens to
|
||||
create your access token and retry with:
|
||||
|
||||
>>> Pipeline.from_pretrained('{model_id}',
|
||||
... use_auth_token=YOUR_AUTH_TOKEN)
|
||||
|
||||
If this still does not work, it might be because the pipeline is gated:
|
||||
visit https://hf.co/{model_id} to accept the user conditions."""
|
||||
)
|
||||
model_id, revision, config_yml = download_from_hf_hub(
|
||||
str(checkpoint),
|
||||
AssetFileName.Pipeline,
|
||||
cache_dir=cache_dir,
|
||||
token=token,
|
||||
)
|
||||
if config_yml is None:
|
||||
return None
|
||||
|
||||
with open(config_yml, "r") as fp:
|
||||
config = yaml.load(fp, Loader=yaml.SafeLoader)
|
||||
|
||||
expand_subfolders(config, model_id, use_auth_token=use_auth_token)
|
||||
expand_subfolders(
|
||||
config, model_id, revision=revision, token=token, cache_dir=cache_dir
|
||||
)
|
||||
|
||||
if "version" in config:
|
||||
check_version(
|
||||
@@ -185,7 +169,8 @@ visit https://hf.co/{model_id} to accept the user conditions."""
|
||||
pipeline_name, default_module_name="pyannote.pipeline.blocks"
|
||||
)
|
||||
params = config["pipeline"].get("params", {})
|
||||
params.setdefault("use_auth_token", use_auth_token)
|
||||
params.setdefault("token", token)
|
||||
params.setdefault("cache_dir", cache_dir)
|
||||
pipeline = Klass(**params)
|
||||
|
||||
# freeze parameters
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, Text, Union
|
||||
|
||||
from pyannote.core import Annotation, SlidingWindowFeature
|
||||
@@ -53,10 +54,10 @@ class MultiLabelSegmentation(Pipeline):
|
||||
Defaults to optimizing identification error rate.
|
||||
share_min_duration : bool, optional
|
||||
If True, `min_duration_on` and `min_duration_off` are shared among labels.
|
||||
use_auth_token : str, optional
|
||||
When loading private huggingface.co models, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
inference_kwargs : dict, optional
|
||||
Keywords arguments passed to Inference.
|
||||
|
||||
@@ -78,7 +79,8 @@ class MultiLabelSegmentation(Pipeline):
|
||||
segmentation: Optional[PipelineModel] = None,
|
||||
fscore: bool = False,
|
||||
share_min_duration: bool = False,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
**inference_kwargs,
|
||||
):
|
||||
|
||||
@@ -94,7 +96,7 @@ class MultiLabelSegmentation(Pipeline):
|
||||
self.share_min_duration = share_min_duration
|
||||
|
||||
# load model
|
||||
model = get_model(segmentation, use_auth_token=use_auth_token)
|
||||
model = get_model(segmentation, token=token, cache_dir=cache_dir)
|
||||
|
||||
self._classes = model.specifications.classes
|
||||
self._segmentation = Inference(model, **inference_kwargs)
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
"""Overlapped speech detection pipelines"""
|
||||
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, Text, Union
|
||||
|
||||
import numpy as np
|
||||
@@ -96,10 +97,10 @@ class OverlappedSpeechDetection(Pipeline):
|
||||
recall : float, optional
|
||||
Optimize precision at target recall
|
||||
Defaults to optimize precision/recall fscore
|
||||
use_auth_token : str, optional
|
||||
When loading private huggingface.co models, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
inference_kwargs : dict, optional
|
||||
Keywords arguments passed to Inference.
|
||||
|
||||
@@ -118,7 +119,8 @@ class OverlappedSpeechDetection(Pipeline):
|
||||
segmentation: PipelineModel = "pyannote/segmentation",
|
||||
precision: Optional[float] = None,
|
||||
recall: Optional[float] = None,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
**inference_kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -126,7 +128,7 @@ class OverlappedSpeechDetection(Pipeline):
|
||||
self.segmentation = segmentation
|
||||
|
||||
# load model
|
||||
model = get_model(segmentation, use_auth_token=use_auth_token)
|
||||
model = get_model(segmentation, token=token, cache_dir=cache_dir)
|
||||
|
||||
if model.dimension > 1:
|
||||
inference_kwargs["pre_aggregation_hook"] = lambda scores: np.partition(
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
"""Resegmentation pipeline"""
|
||||
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, Text, Union
|
||||
|
||||
import numpy as np
|
||||
@@ -67,10 +68,10 @@ class Resegmentation(SpeakerDiarizationMixin, Pipeline):
|
||||
Optimize for a variant of diarization error rate.
|
||||
Defaults to {"collar": 0.0, "skip_overlap": False}. This is used in `get_metric`
|
||||
when instantiating the metric: GreedyDiarizationErrorRate(**der_variant).
|
||||
use_auth_token : str, optional
|
||||
When loading private huggingface.co models, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
|
||||
Hyper-parameters
|
||||
----------------
|
||||
@@ -87,14 +88,15 @@ class Resegmentation(SpeakerDiarizationMixin, Pipeline):
|
||||
segmentation: PipelineModel = "pyannote/segmentation",
|
||||
diarization: Text = "diarization",
|
||||
der_variant: Optional[dict] = None,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.segmentation = segmentation
|
||||
self.diarization = diarization
|
||||
|
||||
model: Model = get_model(segmentation, use_auth_token=use_auth_token)
|
||||
model: Model = get_model(segmentation, token=token, cache_dir=cache_dir)
|
||||
self._segmentation = Inference(model)
|
||||
|
||||
self._audio = model.audio
|
||||
|
||||
@@ -27,6 +27,7 @@ import itertools
|
||||
import math
|
||||
import textwrap
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Callable, Mapping, Optional, Text, Union
|
||||
|
||||
import numpy as np
|
||||
@@ -85,10 +86,10 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
Optimize for a variant of diarization error rate.
|
||||
Defaults to {"collar": 0.0, "skip_overlap": False}. This is used in `get_metric`
|
||||
when instantiating the metric: GreedyDiarizationErrorRate(**der_variant).
|
||||
use_auth_token : str, optional
|
||||
When loading private huggingface.co models, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
|
||||
Usage
|
||||
-----
|
||||
@@ -123,12 +124,13 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
embedding_batch_size: int = 1,
|
||||
segmentation_batch_size: int = 1,
|
||||
der_variant: Optional[dict] = None,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.segmentation_model = segmentation
|
||||
model: Model = get_model(segmentation, use_auth_token=use_auth_token)
|
||||
model: Model = get_model(segmentation, token=token, cache_dir=cache_dir)
|
||||
|
||||
self.segmentation_step = segmentation_step
|
||||
|
||||
@@ -165,7 +167,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
|
||||
else:
|
||||
self._embedding = PretrainedSpeakerEmbedding(
|
||||
self.embedding, use_auth_token=use_auth_token
|
||||
self.embedding, token=token, cache_dir=cache_dir
|
||||
)
|
||||
self._audio = Audio(sample_rate=self._embedding.sample_rate, mono="downmix")
|
||||
metric = self._embedding.metric
|
||||
|
||||
@@ -36,13 +36,10 @@ from torch.nn.utils.rnn import pad_sequence
|
||||
from pyannote.audio import Inference, Model, Pipeline
|
||||
from pyannote.audio.core.inference import BaseInference
|
||||
from pyannote.audio.core.io import AudioFile
|
||||
from pyannote.audio.core.model import CACHE_DIR
|
||||
from pyannote.audio.pipelines.utils import PipelineModel, get_model
|
||||
|
||||
try:
|
||||
from speechbrain.inference import (
|
||||
EncoderClassifier as SpeechBrain_EncoderClassifier,
|
||||
)
|
||||
from speechbrain.inference import EncoderClassifier as SpeechBrain_EncoderClassifier
|
||||
|
||||
SPEECHBRAIN_IS_AVAILABLE = True
|
||||
except ImportError:
|
||||
@@ -211,10 +208,10 @@ class SpeechBrainPretrainedSpeakerEmbedding(BaseInference):
|
||||
Name of SpeechBrain model
|
||||
device : torch.device, optional
|
||||
Device
|
||||
use_auth_token : str, optional
|
||||
When loading private huggingface.co models, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
|
||||
Usage
|
||||
-----
|
||||
@@ -235,7 +232,8 @@ class SpeechBrainPretrainedSpeakerEmbedding(BaseInference):
|
||||
self,
|
||||
embedding: Text = "speechbrain/spkrec-ecapa-voxceleb",
|
||||
device: Optional[torch.device] = None,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
):
|
||||
if not SPEECHBRAIN_IS_AVAILABLE:
|
||||
raise ImportError(
|
||||
@@ -251,13 +249,15 @@ class SpeechBrainPretrainedSpeakerEmbedding(BaseInference):
|
||||
self.embedding = embedding
|
||||
self.revision = None
|
||||
self.device = device or torch.device("cpu")
|
||||
self.use_auth_token = use_auth_token
|
||||
self.token = token
|
||||
self.cache_dir = cache_dir
|
||||
|
||||
self.classifier_ = SpeechBrain_EncoderClassifier.from_hparams(
|
||||
source=self.embedding,
|
||||
savedir=f"{CACHE_DIR}/speechbrain",
|
||||
savedir=f"{self.cache_dir}/speechbrain",
|
||||
run_opts={"device": self.device},
|
||||
use_auth_token=self.use_auth_token,
|
||||
token=self.token,
|
||||
huggingface_cache_dir=self.cache_dir,
|
||||
revision=self.revision,
|
||||
)
|
||||
|
||||
@@ -269,9 +269,10 @@ class SpeechBrainPretrainedSpeakerEmbedding(BaseInference):
|
||||
|
||||
self.classifier_ = SpeechBrain_EncoderClassifier.from_hparams(
|
||||
source=self.embedding,
|
||||
savedir=f"{CACHE_DIR}/speechbrain",
|
||||
savedir=f"{self.cache_dir}/speechbrain",
|
||||
run_opts={"device": device},
|
||||
use_auth_token=self.use_auth_token,
|
||||
token=self.token,
|
||||
huggingface_cache_dir=self.cache_dir,
|
||||
revision=self.revision,
|
||||
)
|
||||
self.device = device
|
||||
@@ -391,6 +392,10 @@ class ONNXWeSpeakerPretrainedSpeakerEmbedding(BaseInference):
|
||||
Path to WeSpeaker pretrained speaker embedding
|
||||
device : torch.device, optional
|
||||
Device
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
|
||||
Usage
|
||||
-----
|
||||
@@ -411,6 +416,8 @@ class ONNXWeSpeakerPretrainedSpeakerEmbedding(BaseInference):
|
||||
self,
|
||||
embedding: Text = "hbredin/wespeaker-voxceleb-resnet34-LM",
|
||||
device: Optional[torch.device] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
):
|
||||
if not ONNX_IS_AVAILABLE:
|
||||
raise ImportError(
|
||||
@@ -420,10 +427,13 @@ class ONNXWeSpeakerPretrainedSpeakerEmbedding(BaseInference):
|
||||
super().__init__()
|
||||
|
||||
if not Path(embedding).exists():
|
||||
|
||||
try:
|
||||
embedding = hf_hub_download(
|
||||
repo_id=embedding,
|
||||
filename="speaker-embedding.onnx",
|
||||
token=token,
|
||||
cache_dir=cache_dir,
|
||||
)
|
||||
except RepositoryNotFoundError:
|
||||
raise ValueError(
|
||||
@@ -618,10 +628,10 @@ class PyannoteAudioPretrainedSpeakerEmbedding(BaseInference):
|
||||
pyannote.audio model
|
||||
device : torch.device, optional
|
||||
Device
|
||||
use_auth_token : str, optional
|
||||
When loading private huggingface.co models, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
|
||||
Usage
|
||||
-----
|
||||
@@ -642,13 +652,14 @@ class PyannoteAudioPretrainedSpeakerEmbedding(BaseInference):
|
||||
self,
|
||||
embedding: PipelineModel = "pyannote/embedding",
|
||||
device: Optional[torch.device] = None,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.embedding = embedding
|
||||
self.device = device or torch.device("cpu")
|
||||
|
||||
self.model_: Model = get_model(self.embedding, use_auth_token=use_auth_token)
|
||||
self.model_: Model = get_model(self.embedding, token=token, cache_dir=cache_dir)
|
||||
self.model_.eval()
|
||||
self.model_.to(self.device)
|
||||
|
||||
@@ -708,7 +719,8 @@ class PyannoteAudioPretrainedSpeakerEmbedding(BaseInference):
|
||||
def PretrainedSpeakerEmbedding(
|
||||
embedding: PipelineModel,
|
||||
device: Optional[torch.device] = None,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
):
|
||||
"""Pretrained speaker embedding
|
||||
|
||||
@@ -719,10 +731,10 @@ def PretrainedSpeakerEmbedding(
|
||||
or a pyannote.audio model.
|
||||
device : torch.device, optional
|
||||
Device
|
||||
use_auth_token : str, optional
|
||||
When loading private huggingface.co models, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
|
||||
Usage
|
||||
-----
|
||||
@@ -743,24 +755,26 @@ def PretrainedSpeakerEmbedding(
|
||||
|
||||
if isinstance(embedding, str) and "pyannote" in embedding:
|
||||
return PyannoteAudioPretrainedSpeakerEmbedding(
|
||||
embedding, device=device, use_auth_token=use_auth_token
|
||||
embedding, device=device, token=token, cache_dir=cache_dir
|
||||
)
|
||||
|
||||
elif isinstance(embedding, str) and "speechbrain" in embedding:
|
||||
return SpeechBrainPretrainedSpeakerEmbedding(
|
||||
embedding, device=device, use_auth_token=use_auth_token
|
||||
embedding, device=device, token=token, cache_dir=cache_dir
|
||||
)
|
||||
|
||||
elif isinstance(embedding, str) and "nvidia" in embedding:
|
||||
return NeMoPretrainedSpeakerEmbedding(embedding, device=device)
|
||||
|
||||
elif isinstance(embedding, str) and "wespeaker" in embedding:
|
||||
return ONNXWeSpeakerPretrainedSpeakerEmbedding(embedding, device=device)
|
||||
return ONNXWeSpeakerPretrainedSpeakerEmbedding(
|
||||
embedding, device=device, token=token, cache_dir=cache_dir
|
||||
)
|
||||
|
||||
else:
|
||||
# fallback to pyannote in case we are loading a local model
|
||||
return PyannoteAudioPretrainedSpeakerEmbedding(
|
||||
embedding, device=device, use_auth_token=use_auth_token
|
||||
embedding, device=device, token=token, cache_dir=cache_dir
|
||||
)
|
||||
|
||||
|
||||
@@ -779,10 +793,10 @@ class SpeakerEmbedding(Pipeline):
|
||||
Pretrained segmentation (or voice activity detection) model.
|
||||
See pyannote.audio.pipelines.utils.get_model for supported format.
|
||||
Defaults to no voice activity detection.
|
||||
use_auth_token : str, optional
|
||||
When loading private huggingface.co models, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
|
||||
Usage
|
||||
-----
|
||||
@@ -798,7 +812,8 @@ class SpeakerEmbedding(Pipeline):
|
||||
self,
|
||||
embedding: PipelineModel = "pyannote/embedding",
|
||||
segmentation: Optional[PipelineModel] = None,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -806,12 +821,12 @@ class SpeakerEmbedding(Pipeline):
|
||||
self.segmentation = segmentation
|
||||
|
||||
self.embedding_model_: Model = get_model(
|
||||
embedding, use_auth_token=use_auth_token
|
||||
embedding, token=token, cache_dir=cache_dir
|
||||
)
|
||||
|
||||
if self.segmentation is not None:
|
||||
segmentation_model: Model = get_model(
|
||||
self.segmentation, use_auth_token=use_auth_token
|
||||
self.segmentation, token=token, cache_dir=cache_dir
|
||||
)
|
||||
self._segmentation = Inference(
|
||||
segmentation_model,
|
||||
|
||||
@@ -27,6 +27,7 @@ import itertools
|
||||
import math
|
||||
import textwrap
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, Text, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
@@ -86,10 +87,10 @@ class SpeechSeparation(SpeakerDiarizationMixin, Pipeline):
|
||||
Optimize for a variant of diarization error rate.
|
||||
Defaults to {"collar": 0.0, "skip_overlap": False}. This is used in `get_metric`
|
||||
when instantiating the metric: GreedyDiarizationErrorRate(**der_variant).
|
||||
use_auth_token : str, optional
|
||||
When loading private huggingface.co models, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
|
||||
Usage
|
||||
-----
|
||||
@@ -133,12 +134,13 @@ class SpeechSeparation(SpeakerDiarizationMixin, Pipeline):
|
||||
embedding_batch_size: int = 1,
|
||||
segmentation_batch_size: int = 1,
|
||||
der_variant: Optional[dict] = None,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.segmentation_model = segmentation
|
||||
model: Model = get_model(segmentation, use_auth_token=use_auth_token)
|
||||
model: Model = get_model(segmentation, token=token, cache_dir=cache_dir)
|
||||
|
||||
self.segmentation_step = segmentation_step
|
||||
|
||||
@@ -175,7 +177,7 @@ class SpeechSeparation(SpeakerDiarizationMixin, Pipeline):
|
||||
|
||||
else:
|
||||
self._embedding = PretrainedSpeakerEmbedding(
|
||||
self.embedding, use_auth_token=use_auth_token
|
||||
self.embedding, token=token, cache_dir=cache_dir
|
||||
)
|
||||
self._audio = Audio(sample_rate=self._embedding.sample_rate, mono="downmix")
|
||||
metric = self._embedding.metric
|
||||
|
||||
@@ -24,12 +24,10 @@ from .diarization import SpeakerDiarizationMixin
|
||||
from .getter import (
|
||||
PipelineAugmentation,
|
||||
PipelineCalibration,
|
||||
PipelineInference,
|
||||
PipelineModel,
|
||||
get_augmentation,
|
||||
get_calibration,
|
||||
get_devices,
|
||||
get_inference,
|
||||
get_model,
|
||||
)
|
||||
from .oracle import oracle_segmentation
|
||||
@@ -40,8 +38,6 @@ __all__ = [
|
||||
"get_augmentation",
|
||||
"PipelineAugmentation",
|
||||
"get_devices",
|
||||
"get_inference",
|
||||
"PipelineInference",
|
||||
"get_model",
|
||||
"PipelineModel",
|
||||
"get_calibration",
|
||||
|
||||
@@ -21,21 +21,23 @@
|
||||
# SOFTWARE.
|
||||
|
||||
import itertools
|
||||
from pathlib import Path
|
||||
from typing import Dict, Mapping, Optional, Text, Union
|
||||
|
||||
import torch
|
||||
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
|
||||
from torch_audiomentations.utils.config import from_dict as augmentation_from_dict
|
||||
|
||||
from pyannote.audio import Inference, Model
|
||||
from pyannote.audio.core.calibration import Calibration
|
||||
from pyannote.audio.core.model import Model
|
||||
|
||||
PipelineModel = Union[Model, Text, Mapping]
|
||||
|
||||
|
||||
def get_model(
|
||||
model: PipelineModel,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
) -> Model:
|
||||
"""Load pretrained model and set it into `eval` mode.
|
||||
|
||||
@@ -46,10 +48,10 @@ def get_model(
|
||||
When `str`, assumes that this is either the path to a checkpoint or the name of a
|
||||
pretrained model on Huggingface.co and loads with `Model.from_pretrained(model)`
|
||||
When `dict`, loads with `Model.from_pretrained(**model)`.
|
||||
use_auth_token : str, optional
|
||||
When loading a private or gated huggingface.co pipeline, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by visiting https://hf.co/settings/tokens
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -73,12 +75,15 @@ def get_model(
|
||||
pass
|
||||
|
||||
elif isinstance(model, Text):
|
||||
model = Model.from_pretrained(
|
||||
model, use_auth_token=use_auth_token, strict=False
|
||||
_model = Model.from_pretrained(
|
||||
model, token=token, cache_dir=cache_dir, strict=False
|
||||
)
|
||||
if _model:
|
||||
model = _model
|
||||
|
||||
elif isinstance(model, Mapping):
|
||||
model.setdefault("use_auth_token", use_auth_token)
|
||||
model.setdefault("token", token)
|
||||
model.setdefault("cache_dir", cache_dir)
|
||||
model = Model.from_pretrained(**model)
|
||||
|
||||
else:
|
||||
@@ -91,54 +96,6 @@ def get_model(
|
||||
return model
|
||||
|
||||
|
||||
PipelineInference = Union[Inference, Model, Text, Mapping]
|
||||
|
||||
|
||||
def get_inference(inference: PipelineInference) -> Inference:
|
||||
"""Load inference
|
||||
|
||||
Parameter
|
||||
---------
|
||||
inference : Inference, Model, str, or dict
|
||||
When `Inference`, returns `inference` as is.
|
||||
When `Model`, wraps it in `Inference(model)`.
|
||||
When `str`, assumes that this is either the path to a checkpoint or the name of a
|
||||
pretrained model on Huggingface.co and loads with `Inference(checkpoint)`.
|
||||
When `dict`, loads with `Inference(**inference)`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
inference : Inference
|
||||
Inference.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> inference = get_inference("hbredin/VoiceActivityDetection-PyanNet-DIHARD")
|
||||
>>> inference = get_inference("/path/to/checkpoint.ckpt")
|
||||
>>> inference = get_inference({"model": "hbredin/VoiceActivityDetection-PyanNet-DIHARD",
|
||||
... "window": "sliding"})
|
||||
|
||||
See also
|
||||
--------
|
||||
pyannote.audio.core.inference.Inference
|
||||
|
||||
"""
|
||||
|
||||
if isinstance(inference, Inference):
|
||||
return inference
|
||||
|
||||
if isinstance(inference, (Model, Text)):
|
||||
return Inference(inference)
|
||||
|
||||
if isinstance(inference, Mapping):
|
||||
return Inference(**inference)
|
||||
|
||||
raise TypeError(
|
||||
f"Unsupported type ({type(inference)}) for loading inference: "
|
||||
f"expected `Model`, `str` or `dict`."
|
||||
)
|
||||
|
||||
|
||||
PipelineAugmentation = Union[BaseWaveformTransform, Mapping]
|
||||
|
||||
|
||||
@@ -147,7 +104,8 @@ PipelineCalibration = Union[Calibration, Text, Dict]
|
||||
|
||||
def get_calibration(
|
||||
calibration: PipelineCalibration,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
) -> Optional[Calibration]:
|
||||
"""Load pretrained calibration
|
||||
|
||||
@@ -158,6 +116,10 @@ def get_calibration(
|
||||
When `str`, assumes that this is either the path to a checkpoint or the name of a
|
||||
pretrained calibration on Huggingface.co and loads with `Calibration.from_pretrained(calibration)`.
|
||||
When `dict`, loads with `Calibration.from_pretrained(**calibration)`.
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -174,11 +136,12 @@ def get_calibration(
|
||||
|
||||
elif isinstance(calibration, Text):
|
||||
loaded_calibration = Calibration.from_pretrained(
|
||||
calibration, use_auth_token=use_auth_token
|
||||
calibration, token=token, cache_dir=cache_dir
|
||||
)
|
||||
|
||||
elif isinstance(calibration, Dict):
|
||||
calibration.setdefault("use_auth_token", use_auth_token)
|
||||
calibration.setdefault("token", token)
|
||||
calibration.setdefault("cache_dir", cache_dir)
|
||||
loaded_calibration = Calibration.from_pretrained(**calibration)
|
||||
|
||||
else:
|
||||
|
||||
@@ -22,37 +22,22 @@
|
||||
|
||||
"""Voice activity detection pipelines"""
|
||||
|
||||
import tempfile
|
||||
from copy import deepcopy
|
||||
from functools import partial
|
||||
from types import MethodType
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, Text, Union
|
||||
|
||||
import numpy as np
|
||||
from pyannote.core import Annotation, SlidingWindowFeature
|
||||
from pyannote.database.protocol import SpeakerDiarizationProtocol
|
||||
from pyannote.metrics.detection import (
|
||||
DetectionErrorRate,
|
||||
DetectionPrecisionRecallFMeasure,
|
||||
)
|
||||
from pyannote.pipeline.parameter import Categorical, Integer, LogUniform, Uniform
|
||||
from pytorch_lightning import Trainer
|
||||
from torch.optim import SGD
|
||||
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
|
||||
from pyannote.pipeline.parameter import Uniform
|
||||
|
||||
from pyannote.audio import Inference
|
||||
from pyannote.audio.core.callback import GraduallyUnfreeze
|
||||
from pyannote.audio.core.io import AudioFile
|
||||
from pyannote.audio.core.pipeline import Pipeline
|
||||
from pyannote.audio.pipelines.utils import (
|
||||
PipelineAugmentation,
|
||||
PipelineInference,
|
||||
PipelineModel,
|
||||
get_augmentation,
|
||||
get_inference,
|
||||
get_model,
|
||||
)
|
||||
from pyannote.audio.tasks import VoiceActivityDetection as VoiceActivityDetectionTask
|
||||
from pyannote.audio.pipelines.utils import PipelineModel, get_model
|
||||
from pyannote.audio.utils.signal import Binarize
|
||||
|
||||
|
||||
@@ -90,10 +75,10 @@ class VoiceActivityDetection(Pipeline):
|
||||
fscore : bool, optional
|
||||
Optimize (precision/recall) fscore. Defaults to optimizing detection
|
||||
error rate.
|
||||
use_auth_token : str, optional
|
||||
When loading private huggingface.co models, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
inference_kwargs : dict, optional
|
||||
Keywords arguments passed to Inference.
|
||||
|
||||
@@ -111,7 +96,8 @@ class VoiceActivityDetection(Pipeline):
|
||||
self,
|
||||
segmentation: PipelineModel = "pyannote/segmentation",
|
||||
fscore: bool = False,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
token: Union[Text, None] = None,
|
||||
cache_dir: Union[Path, Text, None] = None,
|
||||
**inference_kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -120,7 +106,7 @@ class VoiceActivityDetection(Pipeline):
|
||||
self.fscore = fscore
|
||||
|
||||
# load model and send it to GPU (when available and not already on GPU)
|
||||
model = get_model(segmentation, use_auth_token=use_auth_token)
|
||||
model = get_model(segmentation, token=token, cache_dir=cache_dir)
|
||||
|
||||
inference_kwargs["pre_aggregation_hook"] = lambda scores: np.max(
|
||||
scores, axis=-1, keepdims=True
|
||||
@@ -230,158 +216,3 @@ class VoiceActivityDetection(Pipeline):
|
||||
if self.fscore:
|
||||
return "maximize"
|
||||
return "minimize"
|
||||
|
||||
|
||||
class AdaptiveVoiceActivityDetection(Pipeline):
|
||||
"""Adaptive voice activity detection pipeline
|
||||
|
||||
Let M be a pretrained voice activity detection model.
|
||||
|
||||
For each file f, this pipeline starts by applying the model to obtain a first set of
|
||||
speech/non-speech labels.
|
||||
|
||||
Those (automatic, possibly erroneous) labels are then used to fine-tune M on the very
|
||||
same file f into a M_f model, in a self-supervised manner.
|
||||
|
||||
Finally, the fine-tuned model M_f is applied to file f to obtain the final (and
|
||||
hopefully better) speech/non-speech labels.
|
||||
|
||||
During fine-tuning, frames where the pretrained model M is very confident are weighted
|
||||
more than those with lower confidence: the intuition is that the model will use these
|
||||
high confidence regions to adapt to recording conditions (e.g. background noise) and
|
||||
hence will eventually be better on the parts of f where it was initially not quite
|
||||
confident.
|
||||
|
||||
Conversely, to avoid overfitting too much to those high confidence regions, we use
|
||||
data augmentation and freeze all but the final few layers of the pretrained model M.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
segmentation : Model, str, or dict, optional
|
||||
Pretrained segmentation model.
|
||||
Defaults to "hbredin/VoiceActivityDetection-PyanNet-DIHARD".
|
||||
augmentation : BaseWaveformTransform, or dict, optional
|
||||
torch_audiomentations waveform transform, used during fine-tuning.
|
||||
Defaults to no augmentation.
|
||||
fscore : bool, optional
|
||||
Optimize (precision/recall) fscore.
|
||||
Defaults to optimizing detection error rate.
|
||||
|
||||
Hyper-parameters
|
||||
----------------
|
||||
num_epochs : int
|
||||
Number of epochs (where one epoch = going through the file once).
|
||||
batch_size : int
|
||||
Batch size.
|
||||
learning_rate : float
|
||||
Learning rate.
|
||||
|
||||
See also
|
||||
--------
|
||||
pyannote.audio.pipelines.utils.get_inference
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
segmentation: PipelineInference = "hbredin/VoiceActivityDetection-PyanNet-DIHARD",
|
||||
augmentation: Optional[PipelineAugmentation] = None,
|
||||
fscore: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
# pretrained segmentation model
|
||||
self.inference: Inference = get_inference(segmentation)
|
||||
self.augmentation: BaseWaveformTransform = get_augmentation(augmentation)
|
||||
|
||||
self.fscore = fscore
|
||||
|
||||
self.num_epochs = Integer(0, 10)
|
||||
self.batch_size = Categorical([1, 2, 4, 8, 16, 32])
|
||||
self.learning_rate = LogUniform(1e-6, 1)
|
||||
|
||||
def apply(self, file: AudioFile) -> Annotation:
|
||||
# create a copy of file
|
||||
file = dict(file)
|
||||
|
||||
# get segmentation scores from pretrained segmentation model
|
||||
file["seg"] = self.inference(file)
|
||||
|
||||
# infer voice activity detection scores
|
||||
file["vad"] = np.max(file["seg"], axis=1, keepdims=True)
|
||||
|
||||
# apply voice activity detection pipeline with default parameters
|
||||
vad_pipeline = VoiceActivityDetection("vad").instantiate(
|
||||
{
|
||||
"onset": 0.5,
|
||||
"offset": 0.5,
|
||||
"min_duration_on": 0.0,
|
||||
"min_duration_off": 0.0,
|
||||
}
|
||||
)
|
||||
file["annotation"] = vad_pipeline(file)
|
||||
|
||||
# do not fine tune the model if num_epochs is zero
|
||||
if self.num_epochs == 0:
|
||||
return file["annotation"]
|
||||
|
||||
# infer model confidence from segmentation scores
|
||||
# TODO: scale confidence differently (e.g. via an additional binarisation threshold hyper-parameter)
|
||||
file["confidence"] = np.min(
|
||||
np.abs((file["seg"] - 0.5) / 0.5), axis=1, keepdims=True
|
||||
)
|
||||
|
||||
# create a dummy train-only protocol where `file` is the only training file
|
||||
class DummyProtocol(SpeakerDiarizationProtocol):
|
||||
name = "DummyProtocol"
|
||||
|
||||
def train_iter(self):
|
||||
yield file
|
||||
|
||||
vad_task = VoiceActivityDetectionTask(
|
||||
DummyProtocol(),
|
||||
duration=self.inference.duration,
|
||||
weight="confidence",
|
||||
batch_size=self.batch_size,
|
||||
augmentation=self.augmentation,
|
||||
)
|
||||
|
||||
vad_model = deepcopy(self.inference.model)
|
||||
vad_model.task = vad_task
|
||||
|
||||
def configure_optimizers(model):
|
||||
return SGD(model.parameters(), lr=self.learning_rate)
|
||||
|
||||
vad_model.configure_optimizers = MethodType(configure_optimizers, vad_model)
|
||||
|
||||
with tempfile.TemporaryDirectory() as default_root_dir:
|
||||
trainer = Trainer(
|
||||
max_epochs=self.num_epochs,
|
||||
accelerator="gpu",
|
||||
devices=1,
|
||||
callbacks=[GraduallyUnfreeze(epochs_per_stage=self.num_epochs + 1)],
|
||||
enable_checkpointing=False,
|
||||
default_root_dir=default_root_dir,
|
||||
)
|
||||
trainer.fit(vad_model)
|
||||
|
||||
inference = Inference(
|
||||
vad_model,
|
||||
device=self.inference.device,
|
||||
batch_size=self.inference.batch_size,
|
||||
)
|
||||
file["vad"] = inference(file)
|
||||
|
||||
return vad_pipeline(file)
|
||||
|
||||
def get_metric(self) -> Union[DetectionErrorRate, DetectionPrecisionRecallFMeasure]:
|
||||
"""Return new instance of detection metric"""
|
||||
|
||||
if self.fscore:
|
||||
return DetectionPrecisionRecallFMeasure(collar=0.0, skip_overlap=False)
|
||||
|
||||
return DetectionErrorRate(collar=0.0, skip_overlap=False)
|
||||
|
||||
def get_direction(self):
|
||||
if self.fscore:
|
||||
return "maximize"
|
||||
return "minimize"
|
||||
@@ -22,11 +22,13 @@
|
||||
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
from typing import Optional, Tuple, Union
|
||||
|
||||
from huggingface_hub import hf_hub_download
|
||||
from huggingface_hub.utils import HfHubHTTPError
|
||||
|
||||
from pyannote.audio import __version__
|
||||
|
||||
|
||||
# Correspondence between asset_file type
|
||||
# and their filename on Huggingface
|
||||
@@ -42,7 +44,7 @@ def download_from_hf_hub(
|
||||
subfolder: Optional[str] = None,
|
||||
cache_dir: Optional[Union[str, Path]] = None,
|
||||
token: Union[bool, str, None] = None,
|
||||
) -> Optional[str]:
|
||||
) -> Tuple[str, Optional[str], Optional[str]]:
|
||||
"""Download file from Huggingface Hub
|
||||
|
||||
Parameters
|
||||
@@ -52,14 +54,15 @@ def download_from_hf_hub(
|
||||
asset_file : AssetFileName
|
||||
Type of asset file to download.
|
||||
subfolder : str, optional
|
||||
Folder inside the hf.co model repo.
|
||||
token : str, optional
|
||||
When loading a private hf.co model, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
token that can be obtained by running `huggingface-cli login`
|
||||
Folder inside the model repo.
|
||||
token : str or bool, optional
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to model cache directory. Defaults to content of PYANNOTE_CACHE
|
||||
environment variable, or "~/.cache/torch/pyannote" when unset.
|
||||
Path to the folder where cached files are stored.
|
||||
|
||||
See also
|
||||
--------
|
||||
`huggingface_hub.hf_hub_download`
|
||||
"""
|
||||
|
||||
if "@" in checkpoint:
|
||||
@@ -68,15 +71,20 @@ def download_from_hf_hub(
|
||||
model_id, revision = checkpoint, None
|
||||
|
||||
try:
|
||||
return hf_hub_download(
|
||||
return (
|
||||
model_id,
|
||||
asset_file.value,
|
||||
subfolder=subfolder,
|
||||
repo_type="model",
|
||||
revision=revision,
|
||||
library_name="pyannote",
|
||||
cache_dir=cache_dir,
|
||||
token=token,
|
||||
revision,
|
||||
hf_hub_download(
|
||||
model_id,
|
||||
asset_file.value,
|
||||
subfolder=subfolder,
|
||||
repo_type="model",
|
||||
revision=revision,
|
||||
library_name="pyannote",
|
||||
library_version=__version__,
|
||||
cache_dir=cache_dir,
|
||||
token=token,
|
||||
),
|
||||
)
|
||||
except HfHubHTTPError:
|
||||
print(
|
||||
@@ -87,7 +95,7 @@ It might be because the repository is private or gated:
|
||||
* visit https://hf.co/{model_id} to accept user conditions
|
||||
* visit https://hf.co/settings/tokens to create an authentication token
|
||||
* load the {asset_file.name.lower()} with the `token` argument:
|
||||
>>> {asset_file.name}.from_pretrained('{model_id}', use_auth_token=...)
|
||||
>>> {asset_file.name}.from_pretrained('{model_id}', token=...)
|
||||
"""
|
||||
)
|
||||
return
|
||||
raise
|
||||
@@ -13,7 +13,8 @@ HF_SAMPLE_MODEL_ID = "pyannote/ci-segmentation"
|
||||
|
||||
|
||||
def test_hf_download_inference():
|
||||
inference = Inference(HF_SAMPLE_MODEL_ID, device="cpu")
|
||||
model = Model.from_pretrained(HF_SAMPLE_MODEL_ID)
|
||||
inference = Inference(model, device="cpu")
|
||||
assert isinstance(inference, Inference)
|
||||
|
||||
|
||||
|
||||
@@ -197,7 +197,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pyannote.audio import Pipeline\n",
|
||||
"pretrained_pipeline = Pipeline.from_pretrained(\"pyannote/speaker-diarization\", use_auth_token=True) "
|
||||
"pretrained_pipeline = Pipeline.from_pretrained(\"pyannote/speaker-diarization\", token=True) "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -312,7 +312,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pyannote.audio import Model\n",
|
||||
"model = Model.from_pretrained(\"pyannote/segmentation\", use_auth_token=True)"
|
||||
"model = Model.from_pretrained(\"pyannote/segmentation\", token=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -333,7 +333,7 @@
|
||||
"source": [
|
||||
"from pyannote.audio import Pipeline\n",
|
||||
"pipeline = Pipeline.from_pretrained(\n",
|
||||
" \"pyannote/speech-separation-ami-1.0\", use_auth_token=True)\n",
|
||||
" \"pyannote/speech-separation-ami-1.0\", token=True)\n",
|
||||
"\n",
|
||||
"# you can safely ignore that various warnings below."
|
||||
]
|
||||
@@ -961,7 +961,7 @@
|
||||
"from pyannote.audio.core.model import Model\n",
|
||||
"\n",
|
||||
"# apply totatonet on the first five seconds\n",
|
||||
"totatonet = Model.from_pretrained(\"pyannote/separation-ami-1.0\", use_auth_token=True)\n",
|
||||
"totatonet = Model.from_pretrained(\"pyannote/separation-ami-1.0\", token=True)\n",
|
||||
"totatonet.to(device)\n",
|
||||
"\n",
|
||||
"cropped_mixture = mixture[:, :sample_rate * 5].unsqueeze(0)\n",
|
||||
|
||||
@@ -413,7 +413,7 @@
|
||||
"source": [
|
||||
"# load pretrained pipeline\n",
|
||||
"from pyannote.audio import Pipeline\n",
|
||||
"pipeline = Pipeline.from_pretrained('pyannote/speaker-diarization-3.1', use_auth_token=True)\n",
|
||||
"pipeline = Pipeline.from_pretrained('pyannote/speaker-diarization-3.1', token=True)\n",
|
||||
"\n",
|
||||
"# send pipeline to GPU (when available)\n",
|
||||
"import torch\n",
|
||||
|
||||
@@ -258,7 +258,7 @@
|
||||
"source": [
|
||||
"from pyannote.audio.core.model import Model\n",
|
||||
"\n",
|
||||
"pretrained_model = Model.from_pretrained(\"pyannote/segmentation-3.0\", use_auth_token=True)\n",
|
||||
"pretrained_model = Model.from_pretrained(\"pyannote/segmentation-3.0\", token=True)\n",
|
||||
"# we assign the OSD task to the model\n",
|
||||
"pretrained_model.task = osd"
|
||||
]
|
||||
|
||||
Reference in new issue
Block a user