feat: add pipelines (and tutorials)

This commit is contained in:
Hervé BREDIN authored and GitHub committed 2020-11-16 15:07:44 +01:00
1 parent 83eeffdaea
commit a9778a4d62
33 files changed
+2132 -80

No files matched your search

+1 -1
View File
@@ -126,7 +126,7 @@
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.tasks.speaker_tracking.task import SpeakerTracking\n",
"from pyannote.audio.tasks import SpeakerTracking\n",
"spk = SpeakerTracking(protocol, duration=2., batch_size=32, num_workers=4)\n",
"model = SimpleSegmentationModel(task=spk)\n",
"trainer = pl.Trainer(max_epochs=1)\n",
+1 -1
View File
@@ -17,7 +17,7 @@
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.tasks.voice_activity_detection.task import VoiceActivityDetection\n",
"from pyannote.audio.tasks import VoiceActivityDetection\n",
"from pyannote.audio.models.debug import SimpleSegmentationModel\n",
"import pytorch_lightning as pl"
]
+1 -1
View File
@@ -40,7 +40,7 @@
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.tasks.voice_activity_detection.task import VoiceActivityDetection\n",
"from pyannote.audio.tasks import VoiceActivityDetection\n",
"from pyannote.audio.models.debug import SimpleSegmentationModel\n",
"import pytorch_lightning as pl"
]
+1 -1
View File
@@ -24,7 +24,7 @@
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.tasks.voice_activity_detection.task import VoiceActivityDetection\n",
"from pyannote.audio.tasks import VoiceActivityDetection\n",
"from pyannote.audio.models.debug import SimpleSegmentationModel\n",
"import pytorch_lightning as pl\n",
"\n",
+1
View File
@@ -375,6 +375,7 @@ class Inference:
chunks of the same duration.
# TODO: document "fixed" better in pyannote.audio.core.io.Audio
# TODO: add support for "Timeline" chunk
Returns
-------
@@ -19,3 +19,14 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from .overlapped_speech_detection import OverlappedSpeechDetection
from .speaker_diarization import SpeakerDiarization
from .voice_activity_detection import VoiceActivityDetection
# from .speaker_change_detection import SpeakerChangeDetection #  not optimizable
# from .speech_turn_assignment import SpeechTurnClosestAssignment # not optimizable
# from .speech_turn_clustering import SpeechTurnClustering #  not optimizable
# from .speech_turn_segmentation import SpeechTurnSegmentation # not optimizable
__all__ = ["VoiceActivityDetection", "OverlappedSpeechDetection", "SpeakerDiarization"]
@@ -0,0 +1,239 @@
# The MIT License (MIT)
#
# Copyright (c) 2018-2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""Overlapped speech detection pipelines"""
from typing import Optional, Text, Union
from pyannote.audio.core.inference import Inference
from pyannote.audio.core.io import AudioFile
from pyannote.audio.utils.signal import Binarize
from pyannote.core import Annotation, Timeline
from pyannote.database import get_annotated
from pyannote.metrics.detection import DetectionPrecisionRecallFMeasure
from pyannote.pipeline import Pipeline
from pyannote.pipeline.parameter import Uniform
def to_overlap(annotation: Annotation) -> Annotation:
"""Get overlapped speech regions
Parameters
----------
annotation : Annotation
Speaker annotation.
Returns
-------
overlap : Annotation
Overlapped speech annotation.
"""
overlap = Timeline(uri=annotation.uri)
for (s1, t1), (s2, t2) in annotation.co_iter(annotation):
l1 = annotation[s1, t1]
l2 = annotation[s2, t2]
if l1 == l2:
continue
overlap.add(s1 & s2)
return overlap.support().to_annotation(generator="string", modality="overlap")
class OracleOverlappedSpeechDetection(Pipeline):
"""Oracle overlapped speech detection pipeline"""
def __call__(self, file: AudioFile) -> Annotation:
"""Return groundtruth overlapped speech detection
Parameter
---------
file : AudioFile
Must provide a "annotation" key.
Returns
-------
hypothesis : Annotation
Overlapped speech regions.
"""
return to_overlap(file["annotation"])
class OverlappedSpeechDetection(Pipeline):
"""Overlapped speech detection pipeline
Parameters
----------
scores : Inference or str
`Inference` instance used to extract raw overlapped speech detection scores.
When `str`, assumes that file already contains a corresponding key with
precomputed scores. Defaults to "osd".
precision : float, optional
Optimize recall at target precision.
Defaults to optimize precision/recall fscore.
recall : float, optional
Optimize precision at target recall
Defaults to optimize precision/recall fscore
Hyper-parameters
----------------
onset, offset : `float`
Onset/offset detection thresholds
min_duration_on, min_duration_off : `float`
Minimum duration in either state (overlap or not)
"""
def __init__(
self,
scores: Union[Inference, Text] = "vad",
precision: Optional[float] = None,
recall: Optional[float] = None,
):
super().__init__()
self.scores = scores
#  hyper-parameters used for hysteresis thresholding
self.onset = Uniform(0.0, 1.0)
self.offset = Uniform(0.0, 1.0)
#  hyper-parameters used for post-processing
# i.e. removing short overlap/non-overlap regions
self.min_duration_on = Uniform(0.0, 2.0)
self.min_duration_off = Uniform(0.0, 2.0)
if (precision is not None) and (recall is not None):
raise ValueError(
"One must choose between optimizing for target precision or target recall."
)
self.precision = precision
self.recall = recall
def initialize(self):
"""Initialize pipeline with current set of parameters"""
self._binarize = Binarize(
onset=self.onset,
offset=self.offset,
min_duration_on=self.min_duration_on,
min_duration_off=self.min_duration_off,
)
def __call__(self, file: AudioFile) -> Annotation:
"""Apply overlapped speech detection
Parameters
----------
file : AudioFile
Processed file.
Returns
-------
speech : Annotation
Overlapped speech regions.
"""
if isinstance(self.scores, Inference):
overlap_probability = self.scores(file)
else:
overlap_probability = file[self.scores]
overlap = self._binarize(overlap_probability)
overlap.uri = file.get("uri", None)
return overlap.to_annotation(generator="string", modality="overlap")
def get_metric(self, **kwargs) -> DetectionPrecisionRecallFMeasure:
"""Get overlapped speech detection metric
Returns
-------
metric : DetectionPrecisionRecallFMeasure
Detection metric.
"""
if (self.precision is not None) or (self.recall is not None):
raise NotImplementedError(
"pyannote.pipeline should use `loss` method fallback."
)
class _Metric(DetectionPrecisionRecallFMeasure):
def compute_components(
_self,
reference: Annotation,
hypothesis: Annotation,
uem: Timeline = None,
**kwargs
) -> dict:
return super().compute_components(
to_overlap(reference), hypothesis, uem=uem, **kwargs
)
return _Metric()
def loss(self, file: AudioFile, hypothesis: Annotation) -> float:
"""Compute recall at target precision (or vice versa)
Parameters
----------
file : AudioFile
Processed file.
hypothesis : Annotation
Hypothesized overlapped speech regions.
Returns
-------
recall (or purity) : float
When optimizing for target precision:
If precision < target_precision, returns (precision - target_precision).
If precision > target_precision, returns recall.
When optimizing for target recall:
If recall < target_recall, returns (recall - target_recall).
If recall > target_recall, returns precision.
"""
fmeasure = DetectionPrecisionRecallFMeasure()
if "overlap_reference" in file:
overlap_reference = file["overlap_reference"]
else:
reference = file["annotation"]
overlap_reference = to_overlap(reference)
file["overlap_reference"] = overlap_reference
_ = fmeasure(overlap_reference, hypothesis, uem=get_annotated(file))
precision, recall, _ = fmeasure.compute_metrics()
if self.precision is not None:
if precision < self.precision:
return precision - self.precision
else:
return recall
elif self.recall is not None:
if recall < self.recall:
return recall - self.recall
else:
return precision
def get_direction(self):
return "maximize"
@@ -0,0 +1,92 @@
# MIT License
#
# Copyright (c) 2018-2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""Speaker change detection pipeline"""
from typing import Text, Union
from pyannote.audio.core.inference import Inference
from pyannote.audio.core.io import AudioFile
from pyannote.audio.utils.signal import Peak
from pyannote.core import Annotation
from pyannote.pipeline import Pipeline
from pyannote.pipeline.parameter import Uniform
class SpeakerChangeDetection(Pipeline):
"""Speaker change detection pipeline
This pipeline is not optimizable because the metric to use for optimization
remains an open problem. It can only be used as a part of larger pipeline.
Parameters
----------
scores : Inference or str, optional
`Inference` instance used to extract raw speaker change detection scores.
When `str`, assumes that file already contains a corresponding key with
precomputed scores. Defaults to "scd".
Hyper-parameters
----------------
alpha : float
Peak detection threshold.
min_duration : float
Segment minimum duration.
"""
def __init__(self, scores: Union[Inference, Text] = "scd"):
super().__init__()
self.scores = scores
# hyper-parameters
self.alpha = Uniform(0.0, 1.0)
self.min_duration = Uniform(0.0, 10.0)
def initialize(self):
"""Initialize pipeline with current set of parameters"""
self._peak = Peak(alpha=self.alpha, min_duration=self.min_duration)
def __call__(self, file: AudioFile) -> Annotation:
"""Apply speaker change detection
Parameters
----------
file : AudioFile
Processed file.
Returns
-------
speech_turns : pyannote.core.Annotation
Speaker homogeneous regions.
"""
if isinstance(self.scores, Inference):
change_probability = self.scores(file)
else:
change_probability = file[self.scores]
# peak detection
change = self._peak(change_probability)
change.uri = file.get("uri", None)
return change.to_annotation(generator="string", modality="audio")
@@ -0,0 +1,246 @@
# The MIT License (MIT)
#
# Copyright (c) 2017-2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import Optional, Text, Union
from pyannote.audio.core.inference import Inference
from pyannote.audio.core.io import AudioFile
from pyannote.core import Annotation
from pyannote.database import get_annotated
from pyannote.metrics.diarization import (
DiarizationPurityCoverageFMeasure,
GreedyDiarizationErrorRate,
)
from pyannote.pipeline import Pipeline
from pyannote.pipeline.parameter import Uniform
from .speech_turn_assignment import SpeechTurnClosestAssignment
from .speech_turn_clustering import SpeechTurnClustering
from .speech_turn_segmentation import SpeechTurnSegmentation
class SpeakerDiarization(Pipeline):
"""Speaker diarization pipeline
Parameters
----------
vad_scores : Inference or str, optional
`Inference` instance used to extract raw voice activity detection scores.
When `str`, assumes that file already contains a corresponding key with
precomputed scores. Defaults to "vad".
scd_scores : Inference or str, optional
`Inference` instance used to extract raw speaker change detection scores.
When `str`, assumes that file already contains a corresponding key with
precomputed scores. Defaults to "scd".
embeddings : Inference or str, optional
`Inference` instance used to extract speaker embeddings. When `str`,
assumes that file already contains a corresponding key with precomputed
embeddings. Defaults to "emb".
metric : {'euclidean', 'cosine', 'angular'}, optional
Metric used for comparing embeddings. Defaults to 'cosine'.
purity : float, optional
Optimize coverage for target purity.
Defaults to optimizing diarization error rate.
coverage : float, optional
Optimize purity for target coverage.
Defaults to optimizing diarization error rate.
fscore : bool, optional
Optimize for purity/coverage fscore.
Defaults to optimizing for diarization error rate.
# TODO: investigate the use of non-weighted fscore/purity/coverage
Hyper-parameters
----------------
cluster_min_duration : float
Do not cluster speech turns shorter than `cluster_min_duration`.
Assign them to the closest cluster (of long speech turns) instead.
"""
def __init__(
self,
vad_scores: Union[Inference, Text] = "vad",
scd_scores: Union[Inference, Text] = "scd",
embeddings: Union[Inference, Text] = "emb",
metric: Optional[str] = "cosine",
purity: Optional[float] = None,
coverage: Optional[float] = None,
fscore: bool = False,
):
super().__init__()
self.vad_scores = vad_scores
self.scd_scores = scd_scores
self.speech_turn_segmentation = SpeechTurnSegmentation(
vad_scores=vad_scores, scd_scores=scd_scores
)
self.embeddings = embeddings
self.metric = metric
self.speech_turn_clustering = SpeechTurnClustering(
embeddings=self.embeddings, metric=self.metric
)
self.speech_turn_assignment = SpeechTurnClosestAssignment(
embeddings=self.embeddings, metric=self.metric
)
# hyper-parameter
self.cluster_min_duration = Uniform(0, 10)
if sum((purity is not None, coverage is not None, fscore)):
raise ValueError(
"One must choose between optimizing for f-score, target purity, or target coverage."
)
self.purity = purity
self.coverage = coverage
self.fscore = fscore
def __call__(self, file: AudioFile) -> Annotation:
"""Apply speaker diarization
Parameters
----------
file : AudioFile
Processed file.
Returns
-------
diarization : Annotation
Speaker diarization
"""
# segmentation into speech turns
speech_turns = self.speech_turn_segmentation(file)
# in case there is one speech turn or less, there is no need to apply
# any kind of clustering approach.
if len(speech_turns) < 2:
return speech_turns
# split short/long speech turns. the idea is to first cluster long
# speech turns (i.e. those for which we can trust embeddings) and then
# assign each speech turn to the closest cluster.
long_speech_turns = speech_turns.empty()
shrt_speech_turns = speech_turns.empty()
for segment, track, label in speech_turns.itertracks(yield_label=True):
if segment.duration < self.cluster_min_duration:
shrt_speech_turns[segment, track] = label
else:
long_speech_turns[segment, track] = label
# in case there are no long speech turn to cluster, we return the
# original speech turns (= shrt_speech_turns)
if len(long_speech_turns) < 1:
return speech_turns
# first: cluster long speech turns
long_speech_turns = self.speech_turn_clustering(file, long_speech_turns)
# then: assign short speech turns to clusters
long_speech_turns.rename_labels(generator="string", copy=False)
if len(shrt_speech_turns) > 0:
shrt_speech_turns.rename_labels(generator="int", copy=False)
shrt_speech_turns = self.speech_turn_assignment(
file, shrt_speech_turns, long_speech_turns
)
# merge short/long speech turns
return long_speech_turns.update(shrt_speech_turns, copy=False).support(
collar=0.0
)
# TODO. add overlap detection
# TODO. add overlap-aware resegmentation
def loss(self, file: AudioFile, hypothesis: Annotation) -> float:
"""Compute coverage at target purity (or vice versa)
Parameters
----------
file : `dict`
File as provided by a pyannote.database protocol.
hypothesis : `pyannote.core.Annotation`
Speech turns.
Returns
-------
coverage (or purity) : float
When optimizing for target purity:
If purity < target_purity, returns (purity - target_purity).
If purity > target_purity, returns coverage.
When optimizing for target coverage:
If coverage < target_coverage, returns (coverage - target_coverage).
If coverage > target_coverage, returns purity.
"""
fmeasure = DiarizationPurityCoverageFMeasure()
reference = file["annotation"]
_ = fmeasure(reference, hypothesis, uem=get_annotated(file))
purity, coverage, _ = fmeasure.compute_metrics()
if self.purity is not None:
if purity > self.purity:
return purity - self.purity
else:
return coverage
elif self.coverage is not None:
if coverage > self.coverage:
return coverage - self.coverage
else:
return purity
def get_metric(
self,
) -> Union[GreedyDiarizationErrorRate, DiarizationPurityCoverageFMeasure]:
"""Return new instance of diarization metric"""
if (self.purity is not None) or (self.coverage is not None):
raise NotImplementedError(
"pyannote.pipeline will use `loss` method fallback."
)
if self.fscore:
return DiarizationPurityCoverageFMeasure(collar=0.0, skip_overlap=False)
# defaults to optimizing diarization error rate
return GreedyDiarizationErrorRate(collar=0.0, skip_overlap=False)
def get_direction(self):
"""Optimization direction"""
if self.purity is not None:
# we maximize coverage at target purity
return "maximize"
elif self.coverage is not None:
# we maximize purity at target coverage
return "maximize"
elif self.fscore:
# we maximize purity/coverage f-score
return "maximize"
else:
# we minimize diarization error rate
return "minimize"
@@ -0,0 +1,113 @@
# The MIT License (MIT)
#
# Copyright (c) 2017-2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import Optional, Text, Union
from pyannote.audio.core.inference import Inference
from pyannote.audio.core.io import AudioFile
from pyannote.core import Annotation
from pyannote.pipeline import Pipeline
from pyannote.pipeline.blocks.classification import ClosestAssignment
from .utils import assert_int_labels, assert_string_labels, gather_label_embeddings
class SpeechTurnClosestAssignment(Pipeline):
"""Assign speech turn to closest cluster
Parameters
----------
embeddings : Inference or str, optional
`Inference` instance used to extract speaker embeddings. When `str`,
assumes that file already contains a corresponding key with precomputed
embeddings. Defaults to "emb".
metric : {'euclidean', 'cosine', 'angular'}, optional
Metric used for comparing embeddings. Defaults to 'cosine'.
"""
def __init__(
self,
embeddings: Union[Inference, Text] = "emb",
metric: Optional[str] = "cosine",
):
super().__init__()
self.embeddings = embeddings
self.metric = metric
self.closest_assignment = ClosestAssignment(metric=self.metric)
def __call__(
self, file: AudioFile, speech_turns: Annotation, targets: Annotation
) -> Annotation:
"""Assign each speech turn to closest target (if close enough)
Parameters
----------
file : AudioFile
Processed file
speech_turns : Annotation
Speech turns. Should only contain `int` labels.
targets : Annotation
Targets. Should only contain `str` labels.
Returns
-------
assigned : Annotation
Assigned speech turns.
"""
assert_string_labels(targets, "targets")
assert_int_labels(speech_turns, "speech_turns")
if isinstance(self.embeddings, Inference):
if self.embeddings.window == "sliding":
# we precompute embeddings using a sliding window
embeddings = self.embeddings(file)
elif self.embeddings.window == "whole":
raise NotImplementedError(
"Inference with 'whole' window is not supported yet."
)
# TODO: warning about speed issue when Inference is on CPU
else:
# we load precomputed embeddings
embeddings = file[self.embeddings]
# gather targets embedding
X_targets, targets_labels, _ = gather_label_embeddings(targets, embeddings)
# gather speech turns embedding
X, assigned_labels, skipped_labels = gather_label_embeddings(
speech_turns, embeddings
)
# assign speech turns to closest class
assignments = self.closest_assignment(X_targets, X)
mapping = {
label: targets_labels[k]
for label, k in zip(assigned_labels, assignments)
if not k < 0
}
return speech_turns.rename_labels(mapping=mapping)
@@ -0,0 +1,110 @@
# The MIT License (MIT)
#
# Copyright (c) 2018-2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import Optional, Text, Union
from pyannote.audio.core.inference import Inference
from pyannote.audio.core.io import AudioFile
from pyannote.core import Annotation
from pyannote.pipeline import Pipeline
from pyannote.pipeline.blocks.clustering import HierarchicalAgglomerativeClustering
from .utils import assert_string_labels, gather_label_embeddings
class SpeechTurnClustering(Pipeline):
"""Speech turn clustering
Parameters
----------
embeddings : Inference or str, optional
`Inference` instance used to extract speaker embeddings. When `str`,
assumes that file already contains a corresponding key with precomputed
embeddings. Defaults to "emb".
metric : {'euclidean', 'cosine', 'angular'}, optional
Metric used for comparing embeddings. Defaults to 'cosine'.
"""
def __init__(
self,
embeddings: Union[Inference, Text] = "emb",
metric: Optional[str] = "cosine",
):
super().__init__()
self.embeddings = embeddings
self.metric = metric
self.clustering = HierarchicalAgglomerativeClustering(
metric=self.metric, use_threshold=True
)
def __call__(self, file: AudioFile, speech_turns: Annotation) -> Annotation:
"""Apply speech turn clustering
Parameters
----------
file : AudioFile
Processed file
speech_turns : Annotation
Speech turns. Should only contain `str` labels.
Returns
-------
speech_turns : Annotation
Clustered speech turns
"""
assert_string_labels(speech_turns, "speech_turns")
if isinstance(self.embeddings, Inference):
if self.embeddings.window == "sliding":
# we precompute embeddings using a sliding window
embeddings = self.embeddings(file)
elif self.embeddings.window == "whole":
raise NotImplementedError(
"Inference with 'whole' window is not supported yet."
)
# TODO: warning about speed issue when Inference is on CPU
else:
# we load precomputed embeddings
embeddings = file[self.embeddings]
X, clustered_labels, skipped_labels = gather_label_embeddings(
speech_turns, embeddings
)
# apply clustering of label embeddings
clusters = self.clustering(X)
# map each clustered label to its cluster (between 1 and N_CLUSTERS)
mapping = {label: k for label, k in zip(clustered_labels, clusters)}
# map each skipped label to its own cluster
# (between -1 and -N_SKIPPED_LABELS)
for i, label in enumerate(skipped_labels):
mapping[label] = -(i + 1)
# do the actual mapping
return speech_turns.rename_labels(mapping=mapping)
@@ -0,0 +1,108 @@
# MIT License
#
# Copyright (c) 2018-2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import Text, Union
from pyannote.audio.core.inference import Inference
from pyannote.audio.core.io import AudioFile
from pyannote.core import Annotation
from pyannote.pipeline import Pipeline
from pyannote.pipeline.parameter import Uniform
from .speaker_change_detection import SpeakerChangeDetection
from .voice_activity_detection import VoiceActivityDetection
class SpeechTurnSegmentation(Pipeline):
"""Speech turn segmentation pipeline
Combines voice activity and speaker change detections
to obtain speaker-homogeneous speech regions.
This pipeline is not optimizable because the metric to use for optimization
remains an open problem. It can only be used as a part of larger pipeline.
Parameters
----------
vad_scores : Inference or str, optional
`Inference` instance used to extract raw voice activity detection scores.
When `str`, assumes that file already contains a corresponding key with
precomputed scores. Defaults to "vad".
scd_scores : Inference or str, optional
`Inference` instance used to extract raw speaker change detection scores.
When `str`, assumes that file already contains a corresponding key with
precomputed scores. Defaults to "scd".
Hyper-parameters
----------------
gap_max_duration : float
Since speaker change detection is trained to detect speaker/speaker changes
but not speaker/non-speech changes, this pipeline considers that a speaker
change should be added in the middle of every non-speech gap, unless the gap
is shorter than `gap_max_duration` seconds AND speaker change did not
actually detect a speaker change during the gap.
Reference
---------
https://gist.github.com/hbredin/caa5468b2b9f22a3ec6f650bfce060e5
"""
def __init__(
self,
vad_scores: Union[Inference, Text] = "vad",
scd_scores: Union[Inference, Text] = "scd",
):
super().__init__()
self.vad_scores = vad_scores
self.scd_scores = scd_scores
self.voice_activity_detection = VoiceActivityDetection(scores=self.vad_scores)
self.speaker_change_detection = SpeakerChangeDetection(scores=self.scd_scores)
self.gap_max_duration = Uniform(0.0, 2.0)
def __call__(self, file: AudioFile) -> Annotation:
"""Apply speech turn segmentation
Parameter
---------
file : AudioFile
Processed file.
Returns
-------
speech_turns : Annotation
Speech turns
"""
vad = self.voice_activity_detection(file).get_timeline()
scd = self.speaker_change_detection(file).get_timeline()
# see https://gist.github.com/hbredin/caa5468b2b9f22a3ec6f650bfce060e5 for details
return (
scd.crop(vad.support(collar=self.gap_max_duration))
.to_annotation(generator="string")
.crop(vad)
)
+114
View File
@@ -0,0 +1,114 @@
# The MIT License (MIT)
#
# Copyright (c) 2017-2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import Union
import numpy as np
from pyannote.audio.core.inference import Inference
from pyannote.core import Annotation, SlidingWindowFeature
def assert_string_labels(annotation: Annotation, name: str):
"""Check that annotation only contains string labels
Parameters
----------
annotation : Annotation
Annotation.
name : str
Name of the annotation (used for user feedback in case of failure)
"""
if any(not isinstance(label, str) for label in annotation.labels()):
msg = f"{name} must contain `str` labels only."
raise ValueError(msg)
def assert_int_labels(annotation: Annotation, name: str):
"""Check that annotation only contains integer labels
Parameters
----------
annotation : Annotation
Annotation.
name : str
Name of the annotation (used for user feedback in case of failure)
"""
if any(not isinstance(label, int) for label in annotation.labels()):
msg = f"{name} must contain `int` labels only."
raise ValueError(msg)
def gather_label_embeddings(
annotation: Annotation, embeddings: Union[SlidingWindowFeature, Inference]
):
"""Extract one embedding per label
Parameters
----------
annotation : Annotation
Annotation
embeddings : SlidingWindowFeature or Inference
Embeddings, either precomputed on a sliding window (SlidingWindowFeature)
or to be computed on the fly (Inference).
Returns
-------
embeddings : ((len(embedded_labels), embedding_dimension) np.ndarray
Embeddings.
embedded_labels : list of labels
Labels for which an embedding has been computed.
skipped_labels : list of labels
Labels for which no embedding could be computed.
"""
X, embedded_labels, skipped_labels = [], [], []
labels = annotation.labels()
for label in labels:
label_support = annotation.label_timeline(label, copy=False)
if isinstance(embeddings, SlidingWindowFeature):
# be more and more permissive until we have
# at least one embedding for current speech turn
for mode in ["strict", "center", "loose"]:
x = embeddings.crop(label_support, mode=mode)
if len(x) > 0:
break
# skip labels so small we don't have any embedding for it
if len(x) < 1:
skipped_labels.append(label)
continue
embedded_labels.append(label)
X.append(np.mean(x, axis=0))
elif isinstance(embeddings, Inference):
# TODO: add "Timeline" chunk support to Inference.crop
raise NotImplementedError()
return np.vstack(X), embedded_labels, skipped_labels
@@ -0,0 +1,141 @@
# MIT License
#
# Copyright (c) 2018-2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""Voice activity detection pipelines"""
from typing import Text, Union
from pyannote.audio.core.inference import Inference
from pyannote.audio.core.io import AudioFile
from pyannote.audio.utils.signal import Binarize
from pyannote.core import Annotation
from pyannote.metrics.detection import (
DetectionErrorRate,
DetectionPrecisionRecallFMeasure,
)
from pyannote.pipeline import Pipeline
from pyannote.pipeline.parameter import Uniform
class OracleVoiceActivityDetection(Pipeline):
"""Oracle voice activity detection pipeline"""
def __call__(self, file: AudioFile) -> Annotation:
"""Return groundtruth voice activity detection
Parameter
---------
file : AudioFile
Must provide a "annotation" key.
Returns
-------
hypothesis : `pyannote.core.Annotation`
Speech regions
"""
speech = file["annotation"].get_timeline().support()
return speech.to_annotation(generator="string", modality="speech")
class VoiceActivityDetection(Pipeline):
"""Voice activity detection pipeline
Parameters
----------
scores : Inference or str, optional
`Inference` instance used to extract raw voice activity detection scores.
When `str`, assumes that file already contains a corresponding key with
precomputed scores. Defaults to "vad".
fscore : bool, optional
Optimize (precision/recall) fscore. Defaults to optimizing detection
error rate.
Hyper-parameters
----------------
onset, offset : float
Onset/offset detection thresholds
min_duration_on, min_duration_off : float
Minimum duration in either state (speech or not)
"""
def __init__(self, scores: Union[Inference, Text] = "vad", fscore: bool = False):
super().__init__()
self.scores = scores
self.fscore = fscore
#  hyper-parameters used for hysteresis thresholding
self.onset = Uniform(0.0, 1.0)
self.offset = Uniform(0.0, 1.0)
#  hyper-parameters used for post-processing
# i.e. removing short speech/non-speech regions
self.min_duration_on = Uniform(0.0, 2.0)
self.min_duration_off = Uniform(0.0, 2.0)
def initialize(self):
"""Initialize pipeline with current set of parameters"""
self._binarize = Binarize(
onset=self.onset,
offset=self.offset,
min_duration_on=self.min_duration_on,
min_duration_off=self.min_duration_off,
)
def __call__(self, file: AudioFile) -> Annotation:
"""Apply voice activity detection
Parameters
----------
file : AudioFile
Processed file.
Returns
-------
speech : `pyannote.core.Annotation`
Speech regions.
"""
if isinstance(self.scores, Inference):
speech_probability = self.scores(file)
else:
speech_probability = file[self.scores]
speech = self._binarize(speech_probability)
speech.uri = file.get("uri", None)
return speech.to_annotation(generator="string", modality="speech")
def get_metric(self) -> Union[DetectionErrorRate, DetectionPrecisionRecallFMeasure]:
"""Return new instance of detection metric"""
if self.fscore:
return DetectionPrecisionRecallFMeasure(collar=0.0, skip_overlap=False)
return DetectionErrorRate(collar=0.0, skip_overlap=False)
def get_direction(self):
if self.fscore:
return "maximize"
return "minimize"
+10 -4
View File
@@ -20,14 +20,20 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from .voice_activity_detection.task import VoiceActivityDetection # isort:skip
from .speaker_change_detection.task import SpeakerChangeDetection # isort:skip
from .overlapped_speech_detection.task import OverlappedSpeechDetection # isort:skip
from .multi_task_segmentation.task import MultiTaskSegmentation # isort:skip
from .segmentation.voice_activity_detection import VoiceActivityDetection # isort:skip
from .segmentation.speaker_change_detection import SpeakerChangeDetection # isort:skip
from .segmentation.overlapped_speech_detection import ( # isort:skip
OverlappedSpeechDetection,
)
from .segmentation.multi_task_segmentation import MultiTaskSegmentation # isort:skip
from .segmentation.speaker_tracking import SpeakerTracking # isort:skip
__all__ = [
"VoiceActivityDetection",
"SpeakerChangeDetection",
"OverlappedSpeechDetection",
"MultiTaskSegmentation",
"SpeakerTracking",
]
File renamed without changes.
@@ -33,7 +33,7 @@ from pyannote.audio.tasks import (
SpeakerChangeDetection,
VoiceActivityDetection,
)
from pyannote.audio.tasks.mixins import SegmentationTaskMixin
from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
from pyannote.audio.utils.random import create_rng_for_worker
from pyannote.core import Segment
from pyannote.database import Protocol
@@ -27,7 +27,7 @@ import numpy as np
from pyannote.audio.core.io import Audio
from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification
from pyannote.audio.tasks.mixins import SegmentationTaskMixin
from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
from pyannote.audio.utils.random import create_rng_for_worker
from pyannote.core import Segment
from pyannote.database import Protocol
@@ -24,7 +24,7 @@ import numpy as np
import scipy.signal
from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification
from pyannote.audio.tasks.mixins import SegmentationTaskMixin
from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
from pyannote.database import Protocol
@@ -23,7 +23,7 @@
from typing import List, Text
from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification
from pyannote.audio.tasks.mixins import SegmentationTaskMixin
from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
from pyannote.database import Protocol
@@ -24,7 +24,7 @@
import numpy as np
from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification
from pyannote.audio.tasks.mixins import SegmentationTaskMixin
from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
from pyannote.database import Protocol
@@ -1,21 +0,0 @@
# MIT License
#
# Copyright (c) 2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
Whitespace-only changes.
@@ -48,6 +48,10 @@ class SpeakerEmbeddingArcFace(Task):
"""
#  TODO: add a ".metric" property that tells how speaker embedding trained with this approach
#  should be compared. could be a string like "cosine" or "euclidean" or a pdist/cdist-like
#  callable. this ".metric" property should be propagated all the way to Inference (via the model).
def __init__(
self,
protocol: Protocol,
@@ -1,21 +0,0 @@
# MIT License
#
# Copyright (c) 2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
@@ -1,21 +0,0 @@
# MIT License
#
# Copyright (c) 2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
+262
View File
@@ -0,0 +1,262 @@
#!/usr/bin/env python
# encoding: utf-8
#
# The MIT License (MIT)
#
# Copyright (c) 2016-2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr
"""
# Signal processing
"""
from typing import List, Literal
import numpy as np
import scipy.signal
from pyannote.core import Segment, SlidingWindowFeature, Timeline
from pyannote.core.utils.generators import pairwise
ThresholdScale = Literal["absolute", "relative", "percentile"]
def scale_threshold(
thresholds: List[float],
scores: SlidingWindowFeature = None,
scale: ThresholdScale = "absolute",
) -> List[float]:
"""
Parameters
----------
thresholds : list of float
Thresholds.
scores : SlidingWindowFeature, optional
One-dimensional scores.
scale : {"absolute", "relative", "percentile"}, optional
Set to "relative" to scale threshold relatively to min/max.
Set to "percentile" to make them relative to 1% and 99% percentiles.
Defaults to "absolute" (do not scale them)
"""
if scale == "absolute":
mini = 0
maxi = 1
elif scale == "relative":
mini = np.nanmin(scores.data)
maxi = np.nanmax(scores.data)
elif scale == "percentile":
mini = np.nanpercentile(scores.data, 1)
maxi = np.nanpercentile(scores.data, 99)
return [mini + t * (maxi - mini) for t in thresholds]
class Binarize:
"""Binarize detection scores using hysteresis thresholding
Parameters
----------
onset : float, optional
Onset threshold. Defaults to 0.5.
offset : float, optional
Offset threshold. Defaults to 0.5.
scale : {"absolute", "relative", "percentile"}, optional
Set to "relative" to make onset/offset relative to min/max.
Set to "percentile" to make them relative 1% and 99% percentiles.
Defaults to "absolute".
min_duration_on : float, optional
Remove active regions shorter than that many seconds. Defaults to 0s.
min_duration_off : float, optional
Fill inactive regions shorter than that many seconds. Defaults to 0s.
pad_onset : float, optional
Extend active regions by moving their start time by that many seconds.
Defaults to 0s.
pad_offset : float, optional
Extend actiev regions by moving their end time by that many seconds.
Defaults to 0s.
Reference
---------
Gregory Gelly and Jean-Luc Gauvain. "Minimum Word Error Training of
RNN-based Voice Activity Detection", InterSpeech 2015.
"""
def __init__(
self,
onset: float = 0.5,
offset: float = 0.5,
scale: ThresholdScale = "absolute",
min_duration_on: float = 0.0,
min_duration_off: float = 0.0,
pad_onset: float = 0.0,
pad_offset: float = 0.0,
):
super().__init__()
self.onset = onset
self.offset = offset
self.scale = scale
self.pad_onset = pad_onset
self.pad_offset = pad_offset
self.min_duration_on = min_duration_on
self.min_duration_off = min_duration_off
def __call__(self, scores: SlidingWindowFeature):
"""Binarize detection scores
Parameters
----------
scores : SlidingWindowFeature
Detection scores.
Returns
-------
active : Timeline
Active regions.
"""
if scores.dimension != 1:
raise ValueError("Binarize expects one-dimensional scores.")
onset, offset = scale_threshold(
(self.onset, self.offset), scores=scores, scale=self.scale
)
num_frames = len(scores)
frames = scores.sliding_window
timestamps = [frames[i].middle for i in range(num_frames)]
# timeline meant to store 'active' regions
active = Timeline()
# initial state
start = timestamps[0]
is_active = scores[0] > self.onset
for t, y in zip(timestamps[1:], scores[1:]):
# currently active
if is_active:
# switching from active to inactive
if y < offset:
region = Segment(start - self.pad_onset, t + self.pad_offset)
active.add(region)
start = t
is_active = False
# currently inactive
else:
# switching from inactive to active
if y > onset:
start = t
is_active = True
# if active at the end, add final region
if is_active:
region = Segment(start - self.pad_onset, t + self.pad_offset)
active.add(region)
# because of padding, some active regions might be overlapping: merge them.
active = active.support()
# remove short active regions
active = Timeline([s for s in active if s.duration > self.min_duration_on])
# fill short inactive regions
inactive = active.gaps()
for s in inactive:
if s.duration < self.min_duration_off:
active.add(s)
return active.support()
class Peak:
"""Peak detection
Parameters
----------
alpha : float, optional
Peak threshold. Defaults to 0.5
scale : {"absolute", "relative", "percentile"}, optional
Set to "relative" to make alpha relative to min/max.
Set to "percentile" to make it relative 1% and 99% percentiles.
Defaults to "absolute".
min_duration : float, optional
Minimum elapsed time between two consecutive peaks. Defaults to 1 second.
"""
def __init__(
self,
alpha: float = 0.5,
min_duration: float = 1.0,
scale: ThresholdScale = "absolute",
):
super(Peak, self).__init__()
self.alpha = alpha
self.scale = scale
self.min_duration = min_duration
def __call__(self, scores: SlidingWindowFeature):
"""Peak detection
Parameter
---------
scores : SlidingWindowFeature
Detection scores.
Returns
-------
segmentation : Timeline
Partition.
"""
if scores.dimension != 1:
raise ValueError("Peak expects one-dimensional scores.")
num_frames = len(scores)
frames = scores.sliding_window
precision = frames.step
order = max(1, int(np.rint(self.min_duration / precision)))
indices = scipy.signal.argrelmax(scores[:], order=order)[0]
(alpha,) = scale_threshold((self.alpha,), scores=scores, scale=self.scale)
peak_time = np.array([frames[i].middle for i in indices if scores[i] > alpha])
boundaries = np.hstack([[frames[0].start], peak_time, [frames[num_frames].end]])
segmentation = Timeline()
for i, (start, end) in enumerate(pairwise(boundaries)):
segment = Segment(start, end)
segmentation.add(segment)
return segmentation
+2
View File
@@ -2,6 +2,8 @@ einops >= 0.3.0
librosa >= 0.8
pyannote.core >= 4.1
pyannote.database >= 4.0.3
pyannote.metrics >= 3.0.1
pyannote.pipeline >= 1.5.2
pytorch_lightning >= 1.0.0
pytorch_metric_learning >= 0.9.93
semver >= 2.10.2
+1 -2
View File
@@ -8,8 +8,7 @@ from pyannote.audio.models.debug import (
MultiTaskSegmentationModel,
SimpleSegmentationModel,
)
from pyannote.audio.tasks import MultiTaskSegmentation
from pyannote.audio.tasks.voice_activity_detection.task import VoiceActivityDetection
from pyannote.audio.tasks import MultiTaskSegmentation, VoiceActivityDetection
from pyannote.core import SlidingWindowFeature
from pyannote.database import FileFinder, get_protocol
+1 -2
View File
@@ -2,8 +2,7 @@ import pytorch_lightning as pl
import torch
from pyannote.audio.models.debug import SimpleSegmentationModel
from pyannote.audio.tasks.speaker_tracking.task import SpeakerTracking
from pyannote.audio.tasks.voice_activity_detection.task import VoiceActivityDetection
from pyannote.audio.tasks import SpeakerTracking, VoiceActivityDetection
from pyannote.database import FileFinder, get_protocol
+338
View File
@@ -0,0 +1,338 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# TODO: switch to AMI\n",
"PROTOCOL = 'Debug.SpeakerDiarization.Debug'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# TODO: update this tutorial to do fine tuning of a model pretrained on DIHARD"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Overlapped speech detection with `pyannote.audio`\n",
"\n",
"Overlapped speech detection (OSD) is the task of detecting regions where at least two speakers are speaking at the same time. In this notebook, we will train and evaluate an OSD pipeline on Debug database."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.database import get_protocol, FileFinder\n",
"protocol = get_protocol(PROTOCOL, preprocessors={\"audio\": FileFinder()})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`pyannote.database` *protocols* usually define \n",
"* a training set: `for training_file in protocol.train(): ...`, \n",
"* a validation set: `for validation_file in protocol.development(): ...` \n",
"* an evaluation set `for evaluation_file in protocol.test(): ...`"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's listen to the first training file and visualize its reference annotation:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"first_training_file = next(protocol.train())"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.utils.preview import listen\n",
"listen(first_training_file)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"first_training_file['annotation']"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The expected output of a perfect overlapped speech detection pipeline would look like this:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.pipelines.overlapped_speech_detection import OracleOverlappedSpeechDetection\n",
"oracle_osd = OracleOverlappedSpeechDetection()\n",
"\n",
"oracle_osd(first_training_file).get_timeline()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Training"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We initialize an OSD *task* that describes how the model will be trained:\n",
"\n",
"* `protocol` indicates that we will use files available in `protocol.train()`.\n",
"* `duration=2.` and `batch_size=16` indicates that the model will ingest batches of 16 two seconds long audio chunks."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.tasks import OverlappedSpeechDetection\n",
"osd = OverlappedSpeechDetection(protocol, duration=2., batch_size=16)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We initialize the *model*: it needs to know about the task (`task=osd`) for which it is being trained for:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.models.debug import SimpleSegmentationModel\n",
"model = SimpleSegmentationModel(task=osd)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now that everything is ready, let's train with `pytorch-ligthning`!"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pytorch_lightning as pl\n",
"trainer = pl.Trainer(max_epochs=10)\n",
"trainer.fit(model, osd)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Inference\n",
"\n",
"Once trained, we will apply the model on a test file:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"test_file = next(protocol.test())\n",
"# here we use a test file provided by the protocol, but it could be any audio file\n",
"# e.g. test_file = \"/path/to/test.wav\"."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Because the model was trained on 2s audio chunks and that test files are likely to be much longer than that, we wrap the `model` with an `Inference` instance: it will take care of sliding a 2s window over the whole file and aggregate the output of the model."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.core.inference import Inference\n",
"inference = Inference(model)\n",
"osd_probability = inference(test_file)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"osd_probability"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Pipeline"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Almost there! To obtain the final overlapped speech regions, we need to apply a detection threshold. \n",
"For that, we rely on the overlapped speech detection pipeline whose hyper-parameters are set manually:\n",
"- `onset=0.5`: mark region as `active` when probability goes above 0.5\n",
"- `offset=0.5`: switch back to `inactive` when probability goes below 0.5\n",
"- `min_duration_on=0.1`: remove `active` regions shorter than 100ms\n",
"- `min_duration_off=0.1`: fill `inactive` regions shorter than 100ms."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.pipelines import OverlappedSpeechDetection as OverlappedSpeechDetectionPipeline\n",
"pipeline = OverlappedSpeechDetectionPipeline(scores=inference).instantiate(\n",
" {\"onset\": 0.5, \"offset\": 0.5, \"min_duration_on\": 0.1, \"min_duration_off\": 0.1})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here we go:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pipeline(test_file).get_timeline()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Optimizing pipeline hyper-parameters"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"W can try to optimize the hyper-parameters (that we chose manually above) on the validation set to get better performance."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# to make things faster, we run the inference once and for all... \n",
"validation_files = list(protocol.development())\n",
"for file in validation_files:\n",
" file['osd'] = inference(file)\n",
"# ... and tell the pipeline to load OSD scores directly from files\n",
"pipeline = OverlappedSpeechDetectionPipeline(scores=\"osd\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.pipeline import Optimizer\n",
"optimizer = Optimizer(pipeline)\n",
"optimizer.tune(validation_files, n_iterations=200, show_progress=False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"There you go: better hyper-parameters that should lead to better results!"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"optimized_pipeline = OverlappedSpeechDetectionPipeline(scores=inference).instantiate(optimizer.best_params)\n",
"optimized_pipeline(test_file).get_timeline()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
+330
View File
@@ -0,0 +1,330 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# TODO: switch to AMI\n",
"PROTOCOL = 'Debug.SpeakerDiarization.Debug'"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Voice activity detection with `pyannote.audio`\n",
"\n",
"Voice activity detection (VAD) is the task of detecting speech regions in a given audio stream or recording. \n",
"In this notebook, we will train and evaluate a VAD pipeline on Debug database."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.database import get_protocol, FileFinder\n",
"protocol = get_protocol(PROTOCOL, preprocessors={\"audio\": FileFinder()})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`pyannote.database` *protocols* usually define \n",
"* a training set: `for training_file in protocol.train(): ...`, \n",
"* a validation set: `for validation_file in protocol.development(): ...` \n",
"* an evaluation set `for evaluation_file in protocol.test(): ...`"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's listen to the first training file and visualize its reference annotation:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"first_training_file = next(protocol.train())"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.utils.preview import listen\n",
"listen(first_training_file)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"first_training_file['annotation']"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The expected output of a perfect voice activity detection pipeline would look like this:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.pipelines.voice_activity_detection import OracleVoiceActivityDetection\n",
"oracle_vad = OracleVoiceActivityDetection()\n",
"\n",
"oracle_vad(first_training_file).get_timeline()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Training"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We initialize a VAD *task* that describes how the model will be trained:\n",
"\n",
"* `protocol` indicates that we will use files available in `protocol.train()`.\n",
"* `duration=2.` and `batch_size=16` indicates that the model will ingest batches of 16 two seconds long audio chunks."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.tasks import VoiceActivityDetection\n",
"vad = VoiceActivityDetection(protocol, duration=2., batch_size=16)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We initialize the *model*: it needs to know about the task (`task=vad`) for which it is being trained for:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.models.debug import SimpleSegmentationModel\n",
"model = SimpleSegmentationModel(task=vad)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now that everything is ready, let's train with `pytorch-ligthning`!"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pytorch_lightning as pl\n",
"trainer = pl.Trainer(max_epochs=10)\n",
"trainer.fit(model, vad)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Inference\n",
"\n",
"Once trained, we will apply the model on a test file:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"test_file = next(protocol.test())\n",
"# here we use a test file provided by the protocol, but it could be any audio file\n",
"# e.g. test_file = \"/path/to/test.wav\"."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Because the model was trained on 2s audio chunks and that test files are likely to be much longer than that, we wrap the `model` with an `Inference` instance: it will take care of sliding a 2s window over the whole file and aggregate the output of the model."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.core.inference import Inference\n",
"inference = Inference(model)\n",
"vad_probability = inference(test_file)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"vad_probability"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Pipeline"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Almost there! To obtain the final speech regions, we need to apply a detection threshold. \n",
"For that, we rely on the voice activity detection pipeline whose hyper-parameters are set manually:\n",
"- `onset=0.5`: mark region as `active` when probability goes above 0.5\n",
"- `offset=0.5`: switch back to `inactive` when probability goes below 0.5\n",
"- `min_duration_on=0.1`: remove `active` regions shorter than 100ms\n",
"- `min_duration_off=0.1`: fill `inactive` regions shorter than 100ms."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.pipelines import VoiceActivityDetection as VoiceActivityDetectionPipeline\n",
"pipeline = VoiceActivityDetectionPipeline(scores=inference).instantiate(\n",
" {\"onset\": 0.5, \"offset\": 0.5, \"min_duration_on\": 0.1, \"min_duration_off\": 0.1})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here we go:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pipeline(test_file).get_timeline()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Optimizing pipeline hyper-parameters"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"While good enough, the hyper-parameters that we chose manually, we can try to optimize them on the validation set to get even better performance."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# to make things faster, we run the inference once and for all... \n",
"validation_files = list(protocol.development())\n",
"for file in validation_files:\n",
" file['vad_scores'] = inference(file)\n",
"# ... and tell the pipeline to load VAD scores directly from files\n",
"pipeline = VoiceActivityDetectionPipeline(scores=\"vad_scores\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.pipeline import Optimizer\n",
"optimizer = Optimizer(pipeline)\n",
"optimizer.tune(validation_files, n_iterations=200, show_progress=False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"There you go: better hyper-parameters that should lead to better results!"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"optimized_pipeline = VoiceActivityDetectionPipeline(scores=inference).instantiate(optimizer.best_params)\n",
"optimized_pipeline(test_file).get_timeline()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}