From 9faa8fc8a89527403dbb80f235cc18b8d034d921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20BREDIN?= Date: Mon, 17 Apr 2023 15:32:37 +0200 Subject: [PATCH] setup: update most dependencies (#1323) - setup: switch to torch 2.0+ and lightning 2.0+ - setup: switch to torchaudio 2.0+ and soundfile 0.12+ - setup: switch to pyannote.core 5.0+ and pyannote.database 5.0+ - setup: switch to speechbrain 0.5.14+ - BREAKING(task): rename `Segmentation` task to `SpeakerDiarization` - BREAKING(task): remove support for variable chunk duration - BREAKING(pipeline): remove `SpeakerSegmentation` pipeline (in favor of `SpeakerDiarization` pipeline) - BREAKING(pipeline): remove support `FINCHClustering` and `HiddenMarkovModelClustering` - BREAKING: drop support for Python 3.7 --- .github/workflows/test.yml | 46 +-- CHANGELOG.md | 8 +- README.md | 22 +- doc/source/index.rst | 3 + pyannote/audio/cli/train.py | 2 +- .../cli/train_config/trainer/default.yaml | 11 +- pyannote/audio/core/model.py | 5 +- pyannote/audio/core/task.py | 17 +- pyannote/audio/pipelines/__init__.py | 2 - pyannote/audio/pipelines/clustering.py | 335 +-------------- .../audio/pipelines/speaker_diarization.py | 29 +- .../audio/pipelines/speaker_segmentation.py | 382 ------------------ .../audio/pipelines/speaker_verification.py | 7 +- .../tasks/segmentation/speaker_diarization.py | 3 +- pyannote/audio/utils/metric.py | 6 +- pyannote/audio/utils/powerset.py | 6 +- requirements.txt | 36 +- setup.py | 3 +- tests/tasks/test_reproducibility.py | 2 +- tutorials/intro.ipynb | 9 +- 20 files changed, 75 insertions(+), 859 deletions(-) delete mode 100644 pyannote/audio/pipelines/speaker_segmentation.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52aad494..dbe2b54d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,9 +2,9 @@ name: Tests on: push: - branches: [ develop ] + branches: [develop] pull_request: - branches: [ develop ] + branches: [develop] jobs: build: @@ -13,28 +13,28 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: [3.7, 3.8, 3.9] + python-version: [3.8, 3.9, "3.10"] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install libsndfile - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get install libsndfile1 - - name: Install pyannote.audio - run: | + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install libsndfile + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get install libsndfile1 + - name: Install pyannote.audio + run: | pip install -e .[dev,testing] - - name: Test with pytest - run: | + - name: Test with pytest + run: | export PYANNOTE_DATABASE_CONFIG=$GITHUB_WORKSPACE/tests/data/database.yml pytest --cov-report=xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - file: ./coverage.xml - env_vars: PYTHON - name: codecov-pyannote-audio - fail_ci_if_error: false + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + file: ./coverage.xml + env_vars: PYTHON + name: codecov-pyannote-audio + fail_ci_if_error: false diff --git a/CHANGELOG.md b/CHANGELOG.md index d30e101a..5390032d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,18 @@ ## Version 3.0 (xxxx-xx-xx) - - setup: switch to pyannote.database 5.0 - feat(task): add support for label scope in speaker diarization task (from pyannote.database 5.0) - feat(task): add support for missing classes in multi-label segmentation task (from pyannote.database 5.0) - improve(task): load metadata as tensors rather than pyannote.core instances + - setup: switch to torch 2.0+ and lightning 2.0+ + - setup: switch to torchaudio 2.0+ and soundfile 0.12+ + - setup: switch to pyannote.core 5.0+ and pyannote.database 5.0+ + - setup: switch to speechbrain 0.5.14+ - BREAKING(task): rename `Segmentation` task to `SpeakerDiarization` - BREAKING(task): remove support for variable chunk duration + - BREAKING(pipeline): remove `SpeakerSegmentation` pipeline (in favor of `SpeakerDiarization` pipeline) + - BREAKING(pipeline): remove support `FINCHClustering` and `HiddenMarkovModelClustering` + - BREAKING: drop support for Python 3.7 ## Version 2.1.1 (2022-10-27) diff --git a/README.md b/README.md index c7136179..c3f9a8dc 100644 --- a/README.md +++ b/README.md @@ -30,31 +30,21 @@ for turn, _, speaker in diarization.itertracks(yield_label=True): # ... ``` -## What's new in `pyannote.audio` 2.x? +## Highlights -For version 2.x of `pyannote.audio`, [I](https://herve.niderb.fr) decided to rewrite almost everything from scratch. -Highlights of this release are: - -- :exploding_head: much better performance (see [Benchmark](#benchmark)) -- :snake: Python-first API - :hugs: pretrained [pipelines](https://hf.co/models?other=pyannote-audio-pipeline) (and [models](https://hf.co/models?other=pyannote-audio-model)) on [:hugs: model hub](https://huggingface.co/pyannote) +- :exploding_head: state-of-the-art performance (see [Benchmark](#benchmark)) +- :snake: Python-first API - :zap: multi-GPU training with [pytorch-lightning](https://pytorchlightning.ai/) - :control_knobs: data augmentation with [torch-audiomentations](https://github.com/asteroid-team/torch-audiomentations) -- :boom: [Prodigy](https://prodi.gy/) recipes for model-assisted audio annotation ## Installation -Only Python 3.8+ is officially supported (though it might work with Python 3.7) +Only Python 3.8+ is supported. ```bash -conda create -n pyannote python=3.8 -conda activate pyannote - -# pytorch 1.11 is required for speechbrain compatibility -# (see https://pytorch.org/get-started/previous-versions/#v1110) -conda install pytorch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0 -c pytorch - -pip install pyannote.audio +# install from develop branch +pip install -qq https://github.com/pyannote/pyannote-audio/archive/refs/heads/develop.zip ``` ## Documentation diff --git a/doc/source/index.rst b/doc/source/index.rst index 69fabbf0..fae1d780 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -9,6 +9,9 @@ Installation :: + $ conda create -n pyannote python=3.10 + $ conda activate pyannote + $ conda install pytorch torchvision torchaudio -c pytorch $ pip install pyannote.audio diff --git a/pyannote/audio/cli/train.py b/pyannote/audio/cli/train.py index be9596e4..d8155c1b 100644 --- a/pyannote/audio/cli/train.py +++ b/pyannote/audio/cli/train.py @@ -26,7 +26,7 @@ from typing import Optional import hydra from hydra.utils import instantiate -from lightning_lite.utilities.seed import seed_everything +from lightning.pytorch import seed_everything from omegaconf import DictConfig, OmegaConf # from pyannote.audio.core.callback import GraduallyUnfreeze diff --git a/pyannote/audio/cli/train_config/trainer/default.yaml b/pyannote/audio/cli/train_config/trainer/default.yaml index fee07869..ac3a60ff 100644 --- a/pyannote/audio/cli/train_config/trainer/default.yaml +++ b/pyannote/audio/cli/train_config/trainer/default.yaml @@ -2,9 +2,7 @@ _target_: pytorch_lightning.Trainer accelerator: auto accumulate_grad_batches: 1 -auto_scale_batch_size: False -auto_lr_find: False -benchmark: False +benchmark: null # TODO: automatically set to True when using fixed duration chunks deterministic: False check_val_every_n_epoch: 1 devices: auto @@ -13,7 +11,7 @@ enable_checkpointing: True enable_model_summary: True enable_progress_bar: True fast_dev_run: False -gradient_clip_val: 0 +gradient_clip_val: null gradient_clip_algorithm: norm limit_predict_batches: 1.0 limit_test_batches: 1.0 @@ -25,16 +23,13 @@ max_steps: -1 max_time: null min_epochs: 1 min_steps: null -move_metrics_to_cpu: False -multiple_trainloader_mode: max_size_cycle num_nodes: 1 num_sanity_val_steps: 2 overfit_batches: 0.0 precision: 32 profiler: null reload_dataloaders_every_n_epochs: 0 -replace_sampler_ddp: True +use_distributed_sampler: True # TODO: check what this does exactly strategy: null sync_batchnorm: False -track_grad_norm: -1 val_check_interval: 1.0 diff --git a/pyannote/audio/core/model.py b/pyannote/audio/core/model.py index 000b469b..ec0a4a25 100644 --- a/pyannote/audio/core/model.py +++ b/pyannote/audio/core/model.py @@ -35,8 +35,8 @@ import torch.nn as nn import torch.optim from huggingface_hub import hf_hub_download from huggingface_hub.utils import RepositoryNotFoundError +from lightning_fabric.utilities.cloud_io import _load as pl_load from pyannote.core import SlidingWindow -from lightning_lite.utilities.cloud_io import _load as pl_load from pytorch_lightning.utilities.model_summary import ModelSummary from semver import VersionInfo from torch.utils.data import DataLoader @@ -523,9 +523,6 @@ class Model(pl.LightningModule): def validation_step(self, batch, batch_idx): return self.task.validation_step(batch, batch_idx) - def validation_epoch_end(self, outputs): - return self.task.validation_epoch_end(outputs) - def configure_optimizers(self): return torch.optim.Adam(self.parameters(), lr=1e-3) diff --git a/pyannote/audio/core/task.py b/pyannote/audio/core/task.py index c7833dc7..df38a49d 100644 --- a/pyannote/audio/core/task.py +++ b/pyannote/audio/core/task.py @@ -23,31 +23,23 @@ from __future__ import annotations -from functools import partial - -import scipy.special - -try: - from functools import cached_property -except ImportError: - from backports.cached_property import cached_property - import multiprocessing import sys import warnings from dataclasses import dataclass from enum import Enum +from functools import cached_property, partial from numbers import Number -from typing import Dict, List, Optional, Sequence, Text, Tuple, Union +from typing import Dict, List, Literal, Optional, Sequence, Text, Tuple, Union import pytorch_lightning as pl +import scipy.special import torch from pyannote.database import Protocol from torch.utils.data import DataLoader, Dataset, IterableDataset from torch_audiomentations import Identity from torch_audiomentations.core.transforms_interface import BaseWaveformTransform from torchmetrics import Metric, MetricCollection -from typing_extensions import Literal from pyannote.audio.utils.loss import binary_cross_entropy, nll_loss from pyannote.audio.utils.protocol import check_protocol @@ -447,9 +439,6 @@ class Task(pl.LightningDataModule): def validation_step(self, batch, batch_idx: int): return self.common_step(batch, batch_idx, "val") - def validation_epoch_end(self, outputs): - pass - def default_metric(self) -> Union[Metric, Sequence[Metric], Dict[str, Metric]]: """Default validation metric""" msg = f"Missing '{self.__class__.__name__}.default_metric' method." diff --git a/pyannote/audio/pipelines/__init__.py b/pyannote/audio/pipelines/__init__.py index d20f9bfb..0c7d2f25 100644 --- a/pyannote/audio/pipelines/__init__.py +++ b/pyannote/audio/pipelines/__init__.py @@ -24,13 +24,11 @@ from .multilabel import MultiLabelSegmentation from .overlapped_speech_detection import OverlappedSpeechDetection from .resegmentation import Resegmentation from .speaker_diarization import SpeakerDiarization -from .speaker_segmentation import SpeakerSegmentation from .voice_activity_detection import VoiceActivityDetection __all__ = [ "VoiceActivityDetection", "OverlappedSpeechDetection", - "SpeakerSegmentation", "SpeakerDiarization", "Resegmentation", "MultiLabelSegmentation", diff --git a/pyannote/audio/pipelines/clustering.py b/pyannote/audio/pipelines/clustering.py index 6258bbd9..f282ea39 100644 --- a/pyannote/audio/pipelines/clustering.py +++ b/pyannote/audio/pipelines/clustering.py @@ -29,33 +29,17 @@ from typing import Tuple import numpy as np from einops import rearrange -from hmmlearn.hmm import GaussianHMM from pyannote.core import SlidingWindow, SlidingWindowFeature from pyannote.pipeline import Pipeline -from pyannote.pipeline.parameter import ( - Categorical, - Integer, - LogUniform, - ParamDict, - Uniform, -) +from pyannote.pipeline.parameter import Categorical, Integer, Uniform from scipy.cluster.hierarchy import fcluster, linkage from scipy.optimize import linear_sum_assignment -from scipy.spatial.distance import cdist, pdist +from scipy.spatial.distance import cdist -from pyannote.audio import Inference from pyannote.audio.core.io import AudioFile from pyannote.audio.pipelines.utils import oracle_segmentation from pyannote.audio.utils.permutation import permutate -try: - from finch import FINCH - - FINCH_IS_AVAILABLE = True - -except ImportError: - FINCH_IS_AVAILABLE = False - class BaseClustering(Pipeline): def __init__( @@ -286,105 +270,6 @@ class BaseClustering(Pipeline): return hard_clusters, soft_clusters -class FINCHClustering(BaseClustering): - """FINCH clustering - - Parameters - ---------- - metric : {"cosine", "euclidean", ...}, optional - Distance metric to use. Defaults to "cosine". - """ - - def __init__( - self, - metric: str = "cosine", - max_num_embeddings: int = np.inf, - constrained_assignment: bool = False, - ): - - if not FINCH_IS_AVAILABLE: - raise ImportError( - "'finch-clust' must be installed to use FINCH clustering. " - "Visit https://pypi.org/project/finch-clust/ for installation instructions." - ) - - super().__init__( - metric=metric, - max_num_embeddings=max_num_embeddings, - constrained_assignment=constrained_assignment, - ) - - self.threshold = Uniform(0.0, 2.0) # assume unit-normalized embeddings - self.method = Categorical(["average", "complete", "single"]) - - def cluster( - self, - embeddings: np.ndarray, - min_clusters: int, - max_clusters: int, - num_clusters: int = None, - ): - """ - - Parameters - ---------- - embeddings : (num_embeddings, dimension) array - Embeddings - min_clusters : int - Minimum number of clusters - max_clusters : int - Maximum number of clusters - num_clusters : int, optional - Actual number of clusters. Default behavior is to estimate it based - on values provided for `min_clusters`, `max_clusters`, and `threshold`. - - Returns - ------- - clusters : (num_embeddings, ) array - 0-indexed cluster indices. - """ - - num_embeddings, _ = embeddings.shape - if num_embeddings == 1: - return np.zeros((1,), dtype=np.uint8) - - # apply FINCH clustering and keep (supposedly pure) penultimate partition - clusters, _, _ = FINCH( - embeddings, - initial_rank=None, - req_clust=None, - distance=self.metric, - ensure_early_exit=True, - verbose=False, - ) - - _, num_partitions = clusters.shape - if num_partitions < 2: - clusters = clusters[:, 0] - else: - clusters = clusters[:, -2] - num_clusters = np.max(clusters) + 1 - - # compute centroids - centroids = np.vstack( - [np.mean(embeddings[clusters == k], axis=0) for k in range(num_clusters)] - ) - - # perform agglomerative clustering on centroids - dendrogram = linkage(centroids, metric=self.metric, method=self.method) - klusters = fcluster(dendrogram, self.threshold, criterion="distance") - 1 - - # update clusters - clusters = -clusters - for i, k in enumerate(klusters): - clusters[clusters == -i] = k - - # TODO: handle min/max/num_clusters - # TODO: handle min_cluster_size - - return clusters - - class AgglomerativeClustering(BaseClustering): """Agglomerative clustering @@ -650,222 +535,6 @@ class OracleClustering(BaseClustering): return hard_clusters, soft_clusters -class HiddenMarkovModelClustering(BaseClustering): - """Hidden Markov Model with Gaussian states""" - - def __init__( - self, - metric: str = "cosine", - constrained_assignment: bool = False, - ): - - if metric not in ["euclidean", "cosine"]: - raise ValueError("`metric` must be one of {'cosine', 'euclidean'}") - - super().__init__( - metric=metric, - constrained_assignment=constrained_assignment, - ) - - self.single_cluster_detection = ParamDict( - quantile=LogUniform(1e-3, 1e-1), - threshold=Uniform(0.0, 2.0), - ) - - self.covariance_type = Categorical(["spherical", "diag", "full", "tied"]) - self.threshold = Uniform(0.0, 2.0) - - def filter_embeddings( - self, embeddings: np.ndarray, segmentations: SlidingWindowFeature - ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - """ - - Parameters - ---------- - embeddings : (num_chunks, num_speakers, dimension) array - Sequence of embeddings. - segmentations : (num_chunks, num_frames, num_speakers) array - Binary segmentations. - - Returns - ------- - train_embeddings : (num_steps, dimension) array - chunk_idx : (num_steps, ) array - speaker_idx : (num_steps, ) array - - """ - - num_chunks, _, _ = embeddings.shape - - # focus on center of each chunk - duration = segmentations.sliding_window.duration - step = segmentations.sliding_window.step - - ratio = 0.5 * (duration - step) / duration - center_segmentations = Inference.trim(segmentations, warm_up=(ratio, ratio)) - # shape: num_chunks, num_center_frames, num_speakers - - # number of frames during which speakers are active - # in the center of the chunk - num_active_frames: np.ndarray = np.sum(center_segmentations.data, axis=1) - # shape: (num_chunks, num_speakers) - - priors = num_active_frames / ( - np.sum(num_active_frames, axis=1, keepdims=True) + 1e-8 - ) - # shape: (num_chunks, local_num_speakers) - - speaker_idx = np.argmax(priors, axis=1) - # (num_chunks, ) - - # TODO: generate alternative sequences that only differs from train_embeddings - # in regions where there is overlap. - - train_embeddings = embeddings[range(num_chunks), speaker_idx] - # (num_chunks, dimension) - - # remove chunks with one of the following property: - # * there is no active speaker in the center of the chunk - # * embedding extraction has failed for the most active speaker in the center of the chunk - center_is_non_speech = np.max(num_active_frames, axis=1) == 0.0 - embedding_is_invalid = np.any(np.isnan(train_embeddings), axis=1) - chunk_idx = np.where(~(embedding_is_invalid | center_is_non_speech))[0] - # (num_chunks, ) - - return (train_embeddings[chunk_idx], chunk_idx, speaker_idx[chunk_idx]) - - def fit_hmm(self, n_components, train_embeddings): - - hmm = GaussianHMM( - n_components=n_components, - covariance_type=self.covariance_type, - n_iter=100, - random_state=42, - implementation="log", - verbose=False, - ) - hmm.fit(train_embeddings) - - return hmm - - def cluster( - self, - embeddings: np.ndarray, - min_clusters: int, - max_clusters: int, - num_clusters: int = None, - ): - - num_embeddings = len(embeddings) - - # FIXME - if max_clusters == num_embeddings: - max_clusters = min(max_clusters, 20) - - if self.metric == "cosine": - # unit-normalize embeddings to somehow make them "euclidean" - with np.errstate(divide="ignore", invalid="ignore"): - euclidean_embeddings = embeddings / np.linalg.norm( - embeddings, axis=-1, keepdims=True - ) - elif self.metric == "euclidean": - euclidean_embeddings = embeddings - - # when the number of clusters is provided, fit a HMM with - # that many states and return the decoded sequence of states - if num_clusters is not None: - hmm = self.fit_hmm(num_clusters, euclidean_embeddings) - - try: - train_clusters = hmm.predict(euclidean_embeddings) - except ValueError: - # ValueError: startprob_ must sum to 1 (got nan) - # TODO: display a warning that something went wrong - train_clusters = np.zeros((num_embeddings,), dtype=np.int8) - - return train_clusters - - # heuristic for detecting cases where there is just one large cluster - # (and a few meaningless outliers) - if min_clusters == 1: - - # Example with quantile = 1% and threshold = 0.4: - # if 99% (100% - 1%) of pairwise distance are smaller than 0.4, - # then we assume that the others are outliers and return one cluster - if ( - np.quantile( - pdist(euclidean_embeddings, metric="euclidean"), - 1.0 - self.single_cluster_detection["quantile"], - ) - < self.single_cluster_detection["threshold"] - ): - - return np.zeros((num_embeddings,), dtype=np.int8) - - # otherwise, we make sure to return at least 2 clusters - min_clusters = max(2, min_clusters) - max_clusters = max(2, max_clusters) - - # fit a HMM with increasing number of states and stop adding - # when the distance between the two closest states - # - either no longer increases - # - or no longer goes above a threshold - # the selected number of states is the last one for which the - # criterion goes above {threshold}. - - # THIS IS A TERRIBLE CRITERION THAT NEEDS TO BE FIXED - - history = [-np.inf] - patience = min(3, max_clusters - min_clusters) - num_clusters = min_clusters - - for n_components in range(min_clusters, max_clusters + 1): - - hmm = self.fit_hmm(n_components, euclidean_embeddings) - try: - train_clusters = hmm.predict(euclidean_embeddings) - except ValueError: # ValueError: startprob_ must sum to 1 (got nan) - # stop adding states as there too many and not enough - # training data to train it in a reliable manner. - break - - # stop early if too few states were found - if len(np.unique(train_clusters)) < n_components: - break - - # compute distance between the two closest centroids - centroids = np.vstack( - [ - np.mean(embeddings[train_clusters == k], axis=0) - for k in range(n_components) - ] - ) - centroids_pdist = pdist(centroids, metric=self.metric) - current_criterion = np.min(centroids_pdist) - - increasing = current_criterion > max(history) - big_enough = current_criterion > self.threshold - - if increasing or big_enough: - num_clusters = n_components - - elif n_components == num_clusters + patience: - break - - history.append(current_criterion) - - hmm = self.fit_hmm(num_clusters, euclidean_embeddings) - try: - train_clusters = hmm.predict(euclidean_embeddings) - except ValueError: - # ValueError: startprob_ must sum to 1 (got nan) - train_clusters = np.zeros((num_embeddings,), dtype=np.int8) - - return train_clusters - - class Clustering(Enum): AgglomerativeClustering = AgglomerativeClustering - FINCHClustering = FINCHClustering - HiddenMarkovModelClustering = HiddenMarkovModelClustering OracleClustering = OracleClustering diff --git a/pyannote/audio/pipelines/speaker_diarization.py b/pyannote/audio/pipelines/speaker_diarization.py index 1e279b82..246e9df7 100644 --- a/pyannote/audio/pipelines/speaker_diarization.py +++ b/pyannote/audio/pipelines/speaker_diarization.py @@ -77,7 +77,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): Defaults (False) to use the whole speech. clustering : str, optional Clustering algorithm. See pyannote.audio.pipelines.clustering.Clustering - for available options. Defaults to "HiddenMarkovModelClustering". + for available options. Defaults to "AgglomerativeClustering". segmentation_batch_size : int, optional Batch size used for speaker segmentation. Defaults to 32. embedding_batch_size : int, optional @@ -112,7 +112,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): segmentation_step: float = 0.1, embedding: PipelineModel = "speechbrain/spkrec-ecapa-voxceleb@5c0be3875fda05e81f3c004ed8c7c06be308de1e", embedding_exclude_overlap: bool = False, - clustering: str = "HiddenMarkovModelClustering", + clustering: str = "AgglomerativeClustering", embedding_batch_size: int = 32, segmentation_batch_size: int = 32, der_variant: dict = None, @@ -177,31 +177,6 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): self.clustering = Klustering.value(metric=metric) def default_parameters(self): - - if ( - self.segmentation_model == "pyannote/segmentation@2022.07" - and self.segmentation_duration == 5.0 - and self.segmentation_step == 0.1 - and self.embedding - == "speechbrain/spkrec-ecapa-voxceleb@5c0be3875fda05e81f3c004ed8c7c06be308de1e" - and self.embedding_exclude_overlap == True - and self.clustering == "HiddenMarkovModelClustering" - ): - return { - "segmentation": { - "threshold": 0.58, - "min_duration_off": 0.0, - }, - "clustering": { - "single_cluster_detection": { - "quantile": 0.05, - "threshold": 1.15, - }, - "covariance_type": "diag", - "threshold": 0.35, - }, - } - raise NotImplementedError() def classes(self): diff --git a/pyannote/audio/pipelines/speaker_segmentation.py b/pyannote/audio/pipelines/speaker_segmentation.py deleted file mode 100644 index 3eeb6070..00000000 --- a/pyannote/audio/pipelines/speaker_segmentation.py +++ /dev/null @@ -1,382 +0,0 @@ -# MIT License -# -# Copyright (c) 2018-2022 CNRS -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -"""Speaker segmentation pipeline""" - -import math -from functools import partial -from typing import Callable, Optional, Text, Union - -import networkx as nx -import numpy as np -from pyannote.core import SlidingWindowFeature -from pyannote.pipeline.parameter import Uniform - -from pyannote.audio.core.inference import Inference -from pyannote.audio.core.io import AudioFile -from pyannote.audio.core.model import Model -from pyannote.audio.core.pipeline import Pipeline -from pyannote.audio.pipelines.utils import ( - PipelineModel, - SpeakerDiarizationMixin, - get_model, -) -from pyannote.audio.utils.metric import ( - DiscreteDiarizationErrorRate, - SlidingDiarizationErrorRate, -) -from pyannote.audio.utils.permutation import mae_cost_func, permutate -from pyannote.audio.utils.signal import binarize - - -class SpeakerSegmentation(SpeakerDiarizationMixin, Pipeline): - """Speaker segmentation - - Parameters - ---------- - segmentation : Model, str, or dict, optional - Pretrained segmentation model. Defaults to "pyannote/segmentation". - See pyannote.audio.pipelines.utils.get_model for supported format. - skip_conversion : bool, optional - Skip final conversion to pyannote.core.Annotation. Defaults to False. - skip_stitching : bool, optional - Skip stitching step. Defaults to False - use_auth_token : str, optional - When loading private huggingface.co models, set `use_auth_token` - to True or to a string containing your hugginface.co authentication - token that can be obtained by running `huggingface-cli login` - - Hyper-parameters - ---------------- - onset : float - Onset speaker activation threshold - offset : float - Offset speaker activation threshold - stitch_threshold : float - Initial stitching threshold. - min_duration_on : float - Remove speaker turn shorter than that many seconds. - min_duration_off : float - Fill same-speaker gaps shorter than that many seconds. - - """ - - def __init__( - self, - segmentation: PipelineModel = "pyannote/segmentation", - skip_conversion: bool = False, - skip_stitching: bool = False, - use_auth_token: Union[Text, None] = None, - ): - super().__init__() - - self.segmentation = segmentation - self.skip_stitching = skip_stitching - self.skip_conversion = skip_conversion - - model: Model = get_model(segmentation, use_auth_token=use_auth_token) - self._segmentation = Inference(model) - self._frames = self._segmentation.model.introspection.frames - - self._audio = model.audio - - # number of speakers in output of segmentation model - self._num_speakers = len(model.specifications.classes) - - #  hyper-parameters used for hysteresis thresholding - self.onset = Uniform(0.0, 1.0) - self.offset = Uniform(0.0, 1.0) - - if not self.skip_stitching: - self.stitch_threshold = Uniform(0.0, 1.0) - - if not (self.skip_stitching or self.skip_conversion): - self.min_duration_on = Uniform(0.0, 1.0) - self.min_duration_off = Uniform(0.0, 1.0) - - self.warm_up = 0.05 - - def default_parameters(self): - # parameters optimized on DIHARD 3 development set - - if self.segmentation == "pyannote/segmentation": - - parameters = { - "onset": 0.84, - "offset": 0.46, - } - - if not self.skip_stitching: - parameters["stitch_threshold"] = 0.39 - - if not (self.skip_stitching or self.skip_conversion): - parameters.update( - { - "min_duration_on": 0.0, - "min_duration_off": 0.0, - } - ) - - return parameters - - raise NotImplementedError() - - CACHED_SEGMENTATION = "cache/segmentation/inference" - - @staticmethod - def get_stitching_graph( - segmentations: SlidingWindowFeature, onset: float = 0.5 - ) -> nx.Graph: - """Build stitching graph - - Parameters - ---------- - segmentations : (num_chunks, num_frames, local_num_speakers)-shaped SlidingWindowFeature - Raw output of segmentation model. - onset : float, optional - Onset speaker activation threshold. Defaults to 0.5 - - Returns - ------- - stitching_graph : nx.Graph - Nodes are (chunk_idx, speaker_idx) tuples. - An edge between two nodes indicate that those are likely to be the same speaker - (the lower the value of "cost" attribute, the more likely). - """ - - chunks = segmentations.sliding_window - num_chunks, num_frames, _ = segmentations.data.shape - max_lookahead = math.floor(chunks.duration / chunks.step - 1) - lookahead = 2 * (max_lookahead,) - - stitching_graph = nx.Graph() - - for C, (chunk, segmentation) in enumerate(segmentations): - for c in range( - max(0, C - lookahead[0]), min(num_chunks, C + lookahead[1] + 1) - ): - - if c == C: - continue - - # extract common temporal support - shift = round((C - c) * num_frames * chunks.step / chunks.duration) - - if shift < 0: - shift = -shift - this_segmentations = segmentation[shift:] - that_segmentations = segmentations[c, : num_frames - shift] - else: - this_segmentations = segmentation[: num_frames - shift] - that_segmentations = segmentations[c, shift:] - - # find the optimal one-to-one mapping - _, (permutation,), (cost,) = permutate( - this_segmentations[np.newaxis], - that_segmentations, - cost_func=mae_cost_func, - return_cost=True, - ) - - for this, that in enumerate(permutation): - - this_is_active = np.any(this_segmentations[:, this] > onset) - that_is_active = np.any(that_segmentations[:, that] > onset) - - if this_is_active: - stitching_graph.add_node((C, this)) - - if that_is_active: - stitching_graph.add_node((c, that)) - - if this_is_active and that_is_active: - stitching_graph.add_edge( - (C, this), (c, that), cost=cost[this, that] - ) - - return stitching_graph - - @staticmethod - def stitchable_components( - stitching_graph: nx.Graph, stitch_threshold: float, factor: float = 0.5 - ): - """Find stitchable connected components - - A component is 'stitchable' if it contains at most one node per chunk - - Parameters - ---------- - stitching_graph : nx.Graph - Stitching graph. - stitch_threshold : float - - Yields - ------ - component : list of (chunk_idx, speaker_idx) tuple - - """ - - f = stitching_graph.copy() - while f: - f.remove_edges_from( - [ - (n1, n2) - for n1, n2, cost in f.edges(data="cost") - if cost > stitch_threshold - ] - ) - for component in list(nx.connected_components(f)): - if len(set(c for c, _ in component)) == len(component): - yield component - f.remove_nodes_from(component) - stitch_threshold *= factor - - def apply( - self, file: AudioFile, hook: Optional[Callable] = None - ) -> SlidingWindowFeature: - """Apply speaker segmentation - - Parameters - ---------- - file : AudioFile - Processed file. - hook : callable, optional - Callback called after each major steps of the pipeline as follows: - hook(step_name, # human-readable name of current step - step_artefact, # artifact generated by current step - file=file) # file being processed - Time-consuming steps call `hook` multiple times with the same `step_name` - and additional `completed` and `total` keyword arguments usable to track - progress of current step. - - Returns - ------- - segmentation : Annotation - Speaker segmentation - """ - - hook = self.setup_hook(file, hook=hook) - - # apply segmentation model (only if needed) - # output shape is (num_chunks, num_frames, local_num_speakers) - if self.training: - if self.CACHED_SEGMENTATION in file: - segmentations = file[self.CACHED_SEGMENTATION] - else: - segmentations = self._segmentation( - file, hook=partial(hook, "segmentation", None) - ) - file[self.CACHED_SEGMENTATION] = segmentations - else: - segmentations: SlidingWindowFeature = self._segmentation( - file, hook=partial(hook, "segmentation", None) - ) - - hook("segmentation", segmentations) - - if self.skip_stitching: - return binarize( - segmentations, onset=self.onset, offset=self.offset, initial_state=False - ) - - # estimate frame-level number of instantaneous speakers - count = self.speaker_count( - segmentations, - onset=self.onset, - offset=self.offset, - warm_up=(self.warm_up, self.warm_up), - frames=self._frames, - ) - hook("speaker_counting", count) - - # trim warm-up regions - segmentations = Inference.trim( - segmentations, warm_up=(self.warm_up, self.warm_up) - ) - - # build stitching graph - stitching_graph = self.get_stitching_graph(segmentations, onset=self.onset) - - # find (constraint-compliant) connected components - components = list( - self.stitchable_components( - stitching_graph, stitch_threshold=self.stitch_threshold - ) - ) - - num_stitches = len(components) - num_chunks, num_frames, _ = segmentations.data.shape - - stitched_segmentations = np.NAN * np.zeros( - (num_chunks, num_frames, num_stitches) - ) - - for k, component in enumerate(components): - for c, s in component: - stitched_segmentations[c, :, k] = segmentations.data[c, :, s] - - stitched_segmentations = SlidingWindowFeature( - stitched_segmentations, segmentations.sliding_window - ) - - hook("stitching", stitched_segmentations) - - # build discrete diarization - discrete_diarization = self.to_diarization(stitched_segmentations, count) - - # remove inactive speakers - discrete_diarization.data = discrete_diarization.data[ - :, np.nonzero(np.sum(discrete_diarization.data, axis=0))[0] - ] - - if self.skip_conversion: - return discrete_diarization - - hook("aggregation", discrete_diarization) - - # convert to continuous diarization - diarization = self.to_annotation( - discrete_diarization, - min_duration_on=self.min_duration_on, - min_duration_off=self.min_duration_off, - ) - - diarization.uri = file["uri"] - - return diarization.rename_labels( - { - label: expected_label - for label, expected_label in zip(diarization.labels(), self.classes()) - } - ) - - def get_metric(self): - - if self.skip_stitching: - return DiscreteDiarizationErrorRate() - - if self.skip_conversion: - raise NotImplementedError( - "Cannot optimize segmentation pipeline when `skip_conversion` is True." - ) - - return SlidingDiarizationErrorRate(window=2.0 * self._segmentation.duration) diff --git a/pyannote/audio/pipelines/speaker_verification.py b/pyannote/audio/pipelines/speaker_verification.py index af32582a..7c73833a 100644 --- a/pyannote/audio/pipelines/speaker_verification.py +++ b/pyannote/audio/pipelines/speaker_verification.py @@ -21,12 +21,7 @@ # SOFTWARE. import warnings - -try: - from functools import cached_property -except ImportError: - from backports.cached_property import cached_property - +from functools import cached_property from typing import Text, Union import numpy as np diff --git a/pyannote/audio/tasks/segmentation/speaker_diarization.py b/pyannote/audio/tasks/segmentation/speaker_diarization.py index 0ab5d6ae..0c7a1fc0 100644 --- a/pyannote/audio/tasks/segmentation/speaker_diarization.py +++ b/pyannote/audio/tasks/segmentation/speaker_diarization.py @@ -23,7 +23,7 @@ import math import warnings from collections import Counter -from typing import Dict, Optional, Sequence, Text, Tuple, Union +from typing import Dict, Literal, Optional, Sequence, Text, Tuple, Union import numpy as np import torch @@ -36,7 +36,6 @@ from pytorch_lightning.loggers import MLFlowLogger, TensorBoardLogger from rich.progress import track from torch_audiomentations.core.transforms_interface import BaseWaveformTransform from torchmetrics import Metric -from typing_extensions import Literal from pyannote.audio.core.task import Problem, Resolution, Specifications, Task from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin diff --git a/pyannote/audio/utils/metric.py b/pyannote/audio/utils/metric.py index 71699da0..d0315204 100644 --- a/pyannote/audio/utils/metric.py +++ b/pyannote/audio/utils/metric.py @@ -20,11 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -try: - from functools import singledispatchmethod -except ImportError: - from singledispatchmethod import singledispatchmethod - +from functools import singledispatchmethod from typing import Dict, List, Optional import numpy as np diff --git a/pyannote/audio/utils/powerset.py b/pyannote/audio/utils/powerset.py index f206b65b..215cb794 100644 --- a/pyannote/audio/utils/powerset.py +++ b/pyannote/audio/utils/powerset.py @@ -24,11 +24,7 @@ # Hervé BREDIN - https://herve.niderb.fr # Alexis PLAQUET -try: - from functools import cached_property -except ImportError: - from backports.cached_property import cached_property - +from functools import cached_property from itertools import combinations import scipy.special diff --git a/requirements.txt b/requirements.txt index 14c07fb7..22e63c5c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,19 @@ -asteroid-filterbanks >=0.4,<0.5 -backports.cached_property -einops >=0.3,<0.4.0 -hmmlearn >=0.2.7,<0.3 -huggingface_hub >= 0.8.1 -networkx >= 2.6,<3.0 +asteroid-filterbanks >=0.4 +einops >=0.6.0 +huggingface_hub >= 0.13.0 +lightning >= 2.0.1 omegaconf >=2.1,<3.0 -pyannote.core >=4.4,<5.0 -pyannote.database >=5.0,<6.0 -pyannote.metrics >=3.2,<4.0 -pyannote.pipeline >=2.3,<3.0 -pytorch_lightning >=1.8.0,<1.9 -pytorch_metric_learning >=1.0.0,<2.0 +pyannote.core >= 5.0.0 +pyannote.database >= 5.0.0 +pyannote.metrics >= 3.2 +pyannote.pipeline >= 2.3 # 2.4 +pytorch_metric_learning >= 2.1.0 rich >= 12.0.0 -semver >=2.10.2,<3.0 -singledispatchmethod -soundfile >=0.10.2,<0.12 -speechbrain >=0.5.12,<0.6 -torch >=1.9 +semver >= 3.0.0 +soundfile >= 0.12.1 +speechbrain >= 0.5.14 +tensorboardX >= 2.6 +torch >= 2.0.0 torch_audiomentations >= 0.11.0 -torchaudio >=0.10,<1.0 -torchmetrics >=0.11,<1.0 -typing_extensions +torchaudio >= 2.0.0 +torchmetrics >= 0.11.0 diff --git a/setup.py b/setup.py index 3fa457d6..4eaa0b05 100644 --- a/setup.py +++ b/setup.py @@ -56,8 +56,9 @@ if __name__ == "__main__": "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Natural Language :: English", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Scientific/Engineering", ], ) diff --git a/tests/tasks/test_reproducibility.py b/tests/tasks/test_reproducibility.py index 5eab714b..a7307e0c 100644 --- a/tests/tasks/test_reproducibility.py +++ b/tests/tasks/test_reproducibility.py @@ -1,5 +1,5 @@ import torch -from lightning_lite.utilities.seed import seed_everything +from lightning.pytorch import seed_everything from pyannote.database import FileFinder, get_protocol from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel diff --git a/tutorials/intro.ipynb b/tutorials/intro.ipynb index f6b42043..8d8bf2ff 100644 --- a/tutorials/intro.ipynb +++ b/tutorials/intro.ipynb @@ -50,14 +50,7 @@ }, "outputs": [], "source": [ - "# for speechbrain\n", - "!pip install -qq torch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0 torchtext==0.12.0\n", - "!pip install -qq speechbrain==0.5.12\n", - "\n", - "# pyannote.audio\n", - "!pip install -qq pyannote.audio\n", - "\n", - "# for visualization purposes\n", + "!pip install -qq https://github.com/pyannote/pyannote-audio/archive/refs/heads/develop.zip\n", "!pip install -qq ipython==7.34.0" ] },