mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
BREAKING: perform channel selection per-file according to 'channel' key
This commit is contained in:
1 parent
0d6f291328
commit
47fa6ac0dd
24 files changed
+705
-95
No files matched your search
@@ -2,8 +2,11 @@
|
||||
|
||||
## next
|
||||
|
||||
- BREAKING(io): rename `Audio.downmix_and_resample` to `Audio.select_channels_and_resample`
|
||||
- BREAKING(io): remove `Audio(mono=...)`; use the file's `channel` policy instead.
|
||||
- BREAKING(cli): remove gap-filling optimization (`--optimize`) and `OptimizedMinDurationOff` outputs from the `benchmark` command
|
||||
- BREAKING: require Python 3.11 or newer; Python 3.10 is no longer supported
|
||||
- feat(task): support `channel: {int | downmix | random | all}` for audio loading (requires `pyannote.database` >= 6.3.0)
|
||||
- fix(model): restore `fbank_only` mode in WeSpeaker embedding models
|
||||
- improve(model): compute WeSpeaker fbank features on MPS instead of falling back to CPU [@Isalia20](https://github.com/Isalia20)
|
||||
- fix(task): avoid training crashes with `spawn`/`forkserver` workers, including
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
" preprocessors={\"audio\": FileFinder()})\n",
|
||||
"\n",
|
||||
"from pyannote.audio.core.io import Audio\n",
|
||||
"audio = Audio(sample_rate=16000, mono=\"downmix\")\n",
|
||||
"audio = Audio(sample_rate=16000)\n",
|
||||
"file = next(protocol.test())\n",
|
||||
"\n",
|
||||
"from pyannote.core import Segment\n",
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ dependencies = [
|
||||
"opentelemetry-sdk>=1.34.0",
|
||||
"opentelemetry-exporter-otlp>=1.34.0",
|
||||
"pyannote-core>=6.1.0",
|
||||
"pyannote-database>=6.1.1",
|
||||
"pyannote-database>=6.3.0",
|
||||
"pyannote-metrics>=4.2.0",
|
||||
"pyannote-pipeline>=4.1.0",
|
||||
"pytorch-metric-learning>=2.8.1",
|
||||
|
||||
@@ -31,7 +31,9 @@ pyannote.audio relies on torchcodec for reading and torchaudio for resampling.
|
||||
import random
|
||||
import warnings
|
||||
from collections.abc import Mapping
|
||||
from enum import StrEnum
|
||||
from io import IOBase
|
||||
from numbers import Integral
|
||||
from pathlib import Path
|
||||
|
||||
import torch.nn.functional as F
|
||||
@@ -56,6 +58,50 @@ except Exception as e:
|
||||
AudioStreamMetadata = None
|
||||
|
||||
|
||||
class Channel(StrEnum):
|
||||
"""Channel selection options"""
|
||||
|
||||
RANDOM = "random"
|
||||
ALL = "all"
|
||||
DOWNMIX = "downmix"
|
||||
|
||||
@classmethod
|
||||
def normalize(cls, channel) -> "int | Channel":
|
||||
"""Normalize channel selection
|
||||
|
||||
Parameters
|
||||
----------
|
||||
channel : int, str, Channel, or None
|
||||
Zero-based channel index or channel selection option.
|
||||
Supported options are "random", "all", and "downmix".
|
||||
None defaults to "downmix".
|
||||
|
||||
Returns
|
||||
-------
|
||||
channel : int or Channel
|
||||
Non-negative Python integer or Channel enum member.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If channel is not a non-negative integer, a supported option, or None.
|
||||
|
||||
"""
|
||||
if channel is None:
|
||||
return cls.DOWNMIX
|
||||
if isinstance(channel, Integral) and not isinstance(channel, bool):
|
||||
if channel >= 0:
|
||||
return int(channel)
|
||||
elif isinstance(channel, str):
|
||||
try:
|
||||
return cls(channel)
|
||||
except ValueError:
|
||||
pass
|
||||
raise ValueError(
|
||||
'channel must be a non-negative integer, "random", "all", or "downmix".'
|
||||
)
|
||||
|
||||
|
||||
AudioFile = str | Path | IOBase | Mapping
|
||||
|
||||
AudioFileDocString = """
|
||||
@@ -68,6 +114,9 @@ Audio files can be provided to the Audio class using different types:
|
||||
|
||||
For last two options, an additional "channel" key can be provided as a zero-indexed
|
||||
integer to load a specific channel: {"audio": "stereo.wav", "channel": 0}
|
||||
or "random" to select a channel randomly on each load or crop. Use "all" to
|
||||
preserve all channels in their original order. Use "downmix" (the default when
|
||||
channel is omitted or None) to average all channels.
|
||||
"""
|
||||
|
||||
|
||||
@@ -114,14 +163,14 @@ class Audio:
|
||||
----------
|
||||
sample_rate: int, optional
|
||||
Target sampling rate. Defaults to using native sampling rate.
|
||||
mono : {'random', 'downmix'}, optional
|
||||
In case of multi-channel audio, convert to single-channel audio
|
||||
using one of the following strategies: select one channel at
|
||||
'random' or 'downmix' by averaging all channels.
|
||||
|
||||
Notes
|
||||
-----
|
||||
Channel selection is controlled by the file dictionary, defaulting to "downmix".
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> audio = Audio(sample_rate=16000, mono='downmix')
|
||||
>>> audio = Audio(sample_rate=16000)
|
||||
>>> waveform, sample_rate = audio({"audio": "/path/to/audio.wav"})
|
||||
>>> assert sample_rate == 16000
|
||||
>>> sample_rate = 44100
|
||||
@@ -215,15 +264,17 @@ class Audio:
|
||||
|
||||
return file
|
||||
|
||||
def __init__(self, sample_rate: int | None = None, mono=None):
|
||||
def __init__(self, sample_rate: int | None = None):
|
||||
super().__init__()
|
||||
self.sample_rate = sample_rate
|
||||
self.mono = mono
|
||||
|
||||
def downmix_and_resample(
|
||||
self, waveform: Tensor, sample_rate: int, channel: int | None = None
|
||||
def select_channels_and_resample(
|
||||
self,
|
||||
waveform: Tensor,
|
||||
sample_rate: int,
|
||||
channel: int | str | Channel | None = None,
|
||||
) -> tuple[Tensor, int]:
|
||||
"""Downmix and resample
|
||||
"""Apply the channel policy and resample
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -231,29 +282,33 @@ class Audio:
|
||||
Waveform.
|
||||
sample_rate : int
|
||||
Sample rate.
|
||||
channel : int, optional
|
||||
Channel to use.
|
||||
channel : int, "random", "all", or "downmix", optional
|
||||
Zero-based channel to use, "random" to select one on each call, or
|
||||
"all" to preserve every channel. "downmix" averages all channels.
|
||||
Defaults to "downmix" when omitted or None.
|
||||
|
||||
Returns
|
||||
-------
|
||||
waveform : (channel, time) Tensor
|
||||
Remixed and resampled waveform
|
||||
Waveform after applying the channel policy and resampling
|
||||
sample_rate : int
|
||||
New sample rate
|
||||
"""
|
||||
|
||||
# select channel if requested
|
||||
if channel is not None:
|
||||
waveform = waveform[channel : channel + 1]
|
||||
|
||||
# downmix to mono
|
||||
channel = Channel.normalize(channel)
|
||||
num_channels = waveform.shape[0]
|
||||
if num_channels > 1:
|
||||
if self.mono == "random":
|
||||
channel = random.randint(0, num_channels - 1)
|
||||
waveform = waveform[channel : channel + 1]
|
||||
elif self.mono == "downmix":
|
||||
waveform = waveform.mean(dim=0, keepdim=True)
|
||||
if num_channels < 1:
|
||||
raise ValueError("Audio must contain at least one channel.")
|
||||
if channel == Channel.RANDOM:
|
||||
channel = random.randint(0, num_channels - 1)
|
||||
if isinstance(channel, int):
|
||||
if channel >= num_channels:
|
||||
raise ValueError(
|
||||
f"Requested channel {channel}, but audio has {num_channels} channels."
|
||||
)
|
||||
waveform = waveform[channel : channel + 1]
|
||||
elif channel == Channel.DOWNMIX and num_channels > 1:
|
||||
waveform = waveform.mean(dim=0, keepdim=True)
|
||||
|
||||
# resample
|
||||
if (self.sample_rate is not None) and (self.sample_rate != sample_rate):
|
||||
@@ -327,7 +382,9 @@ class Audio:
|
||||
waveform = file["waveform"]
|
||||
sample_rate = file["sample_rate"]
|
||||
|
||||
return self.downmix_and_resample(waveform, sample_rate, channel=channel)
|
||||
return self.select_channels_and_resample(
|
||||
waveform, sample_rate, channel=channel
|
||||
)
|
||||
|
||||
if not TORCHCODEC_AVAILABLE:
|
||||
raise RuntimeError(
|
||||
@@ -346,7 +403,7 @@ class Audio:
|
||||
if isinstance(file["audio"], IOBase):
|
||||
file["audio"].seek(0)
|
||||
|
||||
return self.downmix_and_resample(waveform, sample_rate, channel=channel)
|
||||
return self.select_channels_and_resample(waveform, sample_rate, channel=channel)
|
||||
|
||||
def crop(
|
||||
self,
|
||||
@@ -409,7 +466,7 @@ class Audio:
|
||||
|
||||
data = waveform[:, start_sample:end_sample]
|
||||
data = F.pad(data, (pad_start, pad_end))
|
||||
return self.downmix_and_resample(data, sample_rate, channel=channel)
|
||||
return self.select_channels_and_resample(data, sample_rate, channel=channel)
|
||||
|
||||
if not TORCHCODEC_AVAILABLE:
|
||||
raise RuntimeError(
|
||||
@@ -479,4 +536,4 @@ class Audio:
|
||||
pad_end += 1
|
||||
|
||||
data = F.pad(data, (pad_start, pad_end))
|
||||
return self.downmix_and_resample(data, sample_rate, channel=channel)
|
||||
return self.select_channels_and_resample(data, sample_rate, channel=channel)
|
||||
@@ -88,14 +88,13 @@ class Model(lightning.LightningModule):
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
assert num_channels == 1, (
|
||||
"Only mono audio is supported for now (num_channels = 1)"
|
||||
)
|
||||
if not isinstance(num_channels, int) or num_channels < 1:
|
||||
raise ValueError("num_channels must be a positive integer.")
|
||||
|
||||
self.save_hyperparameters("sample_rate", "num_channels")
|
||||
|
||||
self.task = task
|
||||
self.audio = Audio(sample_rate=self.hparams.sample_rate, mono="downmix")
|
||||
self.audio = Audio(sample_rate=self.hparams.sample_rate)
|
||||
|
||||
@property
|
||||
def task(self) -> Task:
|
||||
|
||||
@@ -482,12 +482,9 @@ class Pipeline(_Pipeline):
|
||||
"Cannot preload audio: `waveform` key is already available or will be via a preprocessor."
|
||||
)
|
||||
|
||||
# load waveform in memory (and keep track of its original sample rate)
|
||||
file["waveform"], file["sample_rate"] = Audio()(file)
|
||||
|
||||
# the above line already took care of channel selection,
|
||||
# therefore we remove the `channel` key from the file
|
||||
file.pop("channel", None)
|
||||
# load waveform in memory and keep track of its original sample rate.
|
||||
# keep all channels for now: channel selection is done later at load/crop time.
|
||||
file["waveform"], file["sample_rate"] = Audio()({**file, "channel": "all"})
|
||||
|
||||
return file
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ import warnings
|
||||
from collections import defaultdict
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from enum import Enum, IntEnum
|
||||
from functools import cached_property, partial
|
||||
from numbers import Number
|
||||
from pathlib import Path
|
||||
@@ -50,12 +50,24 @@ from torch_audiomentations import Identity
|
||||
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
|
||||
from torchmetrics import Metric, MetricCollection
|
||||
|
||||
from pyannote.audio.core.io import Channel
|
||||
from pyannote.audio.utils.loss import binary_cross_entropy, nll_loss
|
||||
from pyannote.audio.utils.protocol import check_protocol
|
||||
|
||||
Subsets = list(Subset.__args__)
|
||||
Scopes = list(Scope.__args__)
|
||||
|
||||
MISSING_METADATA = -1
|
||||
|
||||
|
||||
class _CachedChannel(IntEnum):
|
||||
"""Private integer encoding for channel selection in task caches."""
|
||||
|
||||
UNSPECIFIED = MISSING_METADATA
|
||||
RANDOM = -2
|
||||
ALL = -3
|
||||
DOWNMIX = -4
|
||||
|
||||
|
||||
# Type of machine learning problem
|
||||
class Problem(Enum):
|
||||
@@ -332,7 +344,7 @@ class Task(lightning.LightningDataModule):
|
||||
prepared_data = {
|
||||
'protocol': name of the protocol
|
||||
'audio-path': array of N paths to audio
|
||||
'audio-metadata': array of N audio infos such as audio subset, scope and database
|
||||
'audio-metadata': array of N audio infos such as audio channel, subset, scope and database.
|
||||
'audio-annotated': array of N annotated duration (usually equals file duration but might be shorter if file is not fully annotated)
|
||||
'annotations-regions': array of M annotated regions
|
||||
'audio-regions-ids': array of N start/end indices of annotated regions
|
||||
@@ -402,6 +414,16 @@ class Task(lightning.LightningDataModule):
|
||||
# keep track of label scope (file, database, or global)
|
||||
metadatum["scope"] = Scopes.index(file["scope"])
|
||||
|
||||
# Channel selection is a loading option, not categorical metadata.
|
||||
metadatum["channel"] = _CachedChannel.UNSPECIFIED
|
||||
if "channel" in file:
|
||||
channel = Channel.normalize(file["channel"])
|
||||
metadatum["channel"] = (
|
||||
_CachedChannel[channel.name]
|
||||
if isinstance(channel, Channel)
|
||||
else channel
|
||||
)
|
||||
|
||||
remaining_metadata_keys = set(file) - {
|
||||
"uri",
|
||||
"database",
|
||||
@@ -409,6 +431,7 @@ class Task(lightning.LightningDataModule):
|
||||
"audio",
|
||||
"scope",
|
||||
"classes",
|
||||
"channel",
|
||||
"annotation",
|
||||
"annotated",
|
||||
}
|
||||
@@ -525,14 +548,16 @@ class Task(lightning.LightningDataModule):
|
||||
# append segment IDs (start and end time)
|
||||
audio_segments_ids.append((_segments_id, segments_id))
|
||||
|
||||
# since not all metadata keys are present in all files, fallback to -1 when a key is missing
|
||||
# Register channel separately without collecting its possible values.
|
||||
metadata_keys = [*metadata_unique_values, "channel"]
|
||||
# Not all metadata keys are present in every file.
|
||||
metadata = [
|
||||
tuple(metadatum.get(key, -1) for key in metadata_unique_values)
|
||||
tuple(metadatum.get(key, MISSING_METADATA) for key in metadata_keys)
|
||||
for metadatum in metadata
|
||||
]
|
||||
metadata_dtype = [
|
||||
(key, get_dtype(max(m[i] for m in metadata)))
|
||||
for i, key in enumerate(metadata_unique_values)
|
||||
for i, key in enumerate(metadata_keys)
|
||||
]
|
||||
|
||||
# turn list of annotated regions into a single numpy array
|
||||
|
||||
@@ -20,3 +20,32 @@
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
|
||||
from pyannote.audio.core.io import Audio, AudioFile
|
||||
|
||||
|
||||
def _prepare_audio(file: AudioFile) -> dict:
|
||||
"""Apply channel selection before handing audio to a mono-only backend.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file : AudioFile
|
||||
Audio file with an optional channel selection policy.
|
||||
|
||||
Returns
|
||||
-------
|
||||
audio : dict
|
||||
Selected mono waveform as a NumPy array and its sample rate.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If the selected waveform contains more than one channel.
|
||||
"""
|
||||
waveform, sample_rate = Audio()(file)
|
||||
if waveform.shape[0] != 1:
|
||||
raise ValueError(
|
||||
'pyannoteAI backends require mono audio. Use channel="downmix", '
|
||||
'"random", or a fixed channel index instead of "all" for multi-channel audio.'
|
||||
)
|
||||
return {"waveform": waveform.numpy(force=True), "sample_rate": sample_rate}
|
||||
@@ -28,6 +28,7 @@ from pyannote.audio import Pipeline
|
||||
from pyannote.audio.core.io import AudioFile
|
||||
|
||||
from ..speaker_diarization import DiarizeOutput
|
||||
from . import _prepare_audio
|
||||
|
||||
|
||||
class Local(Pipeline):
|
||||
@@ -89,33 +90,20 @@ class Local(Pipeline):
|
||||
-------
|
||||
output : DiarizeOutput
|
||||
DiarizeOutput object containing both regular and exclusive speaker diarization results.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If channel selection leaves multiple channels. The backend requires mono audio.
|
||||
"""
|
||||
|
||||
# if file provides "audio" path
|
||||
if "audio" in file:
|
||||
predictions = self._pipeline.diarize(
|
||||
file["audio"],
|
||||
num_speakers=num_speakers,
|
||||
min_speakers=min_speakers,
|
||||
max_speakers=max_speakers,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# if file provides "waveform", make sure it is numpy (and not torch) array
|
||||
elif "waveform" in file:
|
||||
waveform = file["waveform"]
|
||||
if hasattr(waveform, "numpy"):
|
||||
waveform = waveform.numpy(force=True)
|
||||
|
||||
predictions = self._pipeline.diarize(
|
||||
{"waveform": waveform, "sample_rate": file["sample_rate"]},
|
||||
num_speakers=num_speakers,
|
||||
min_speakers=min_speakers,
|
||||
max_speakers=max_speakers,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
raise ValueError("AudioFile must provide either 'audio' or 'waveform' key")
|
||||
predictions = self._pipeline.diarize(
|
||||
_prepare_audio(file),
|
||||
num_speakers=num_speakers,
|
||||
min_speakers=min_speakers,
|
||||
max_speakers=max_speakers,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
speaker_diarization: Annotation = self._deserialize(predictions["diarization"])
|
||||
exclusive_speaker_diarization: Annotation = self._deserialize(
|
||||
|
||||
@@ -29,6 +29,8 @@ from pyannote.audio import Pipeline
|
||||
from pyannote.audio.core.io import AudioFile
|
||||
from pyannote.audio.pipelines.speaker_diarization import DiarizeOutput
|
||||
|
||||
from . import _prepare_audio
|
||||
|
||||
|
||||
class SDK(Pipeline):
|
||||
"""Wrapper around official pyannoteAI API client
|
||||
@@ -97,6 +99,8 @@ class SDK(Pipeline):
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If channel selection leaves multiple channels. The backend requires mono audio.
|
||||
PyannoteAIFailedJob
|
||||
If the job failed
|
||||
PyannoteAICanceledJob
|
||||
@@ -106,7 +110,7 @@ class SDK(Pipeline):
|
||||
"""
|
||||
|
||||
# upload file to pyannoteAI cloud API
|
||||
media_url: str = self._client.upload(file)
|
||||
media_url: str = self._client.upload(_prepare_audio(file))
|
||||
|
||||
# initiate diarization job
|
||||
job_id = self._client.diarize(
|
||||
|
||||
@@ -268,7 +268,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
self._embedding = PretrainedSpeakerEmbedding(
|
||||
self.embedding, token=token, cache_dir=cache_dir
|
||||
)
|
||||
self._audio = Audio(sample_rate=self._embedding.sample_rate, mono="downmix")
|
||||
self._audio = Audio(sample_rate=self._embedding.sample_rate)
|
||||
metric = self._embedding.metric
|
||||
|
||||
try:
|
||||
|
||||
@@ -180,7 +180,7 @@ class SpeechSeparation(SpeakerDiarizationMixin, Pipeline):
|
||||
self._embedding = PretrainedSpeakerEmbedding(
|
||||
self.embedding, token=token, cache_dir=cache_dir
|
||||
)
|
||||
self._audio = Audio(sample_rate=self._embedding.sample_rate, mono="downmix")
|
||||
self._audio = Audio(sample_rate=self._embedding.sample_rate)
|
||||
metric = self._embedding.metric
|
||||
|
||||
try:
|
||||
|
||||
@@ -61,7 +61,7 @@ def oracle_segmentation(
|
||||
"""
|
||||
|
||||
if "duration" not in file:
|
||||
duration = Audio(mono="downmix").get_duration(file)
|
||||
duration = Audio().get_duration(file)
|
||||
else:
|
||||
duration: float = file["duration"]
|
||||
reference: Annotation = file["annotation"]
|
||||
|
||||
@@ -36,6 +36,7 @@ from torchmetrics import Metric
|
||||
from torchmetrics.classification import BinaryAUROC
|
||||
from tqdm import tqdm
|
||||
|
||||
from pyannote.audio.core.io import Channel
|
||||
from pyannote.audio.core.task import Problem, Resolution, Specifications
|
||||
from pyannote.audio.torchmetrics.classification import EqualErrorRate
|
||||
from pyannote.audio.utils.random import create_rng_for_worker
|
||||
@@ -107,6 +108,7 @@ class SupervisedRepresentationLearningTaskMixin:
|
||||
{
|
||||
"uri": f["uri"],
|
||||
"audio": f["audio"],
|
||||
"channel": Channel.normalize(f.get("channel")),
|
||||
"duration": duration,
|
||||
"speech_turns": speech_turns,
|
||||
}
|
||||
|
||||
@@ -36,7 +36,8 @@ from torch.utils.data._utils.collate import default_collate
|
||||
from torchmetrics import Metric
|
||||
from torchmetrics.classification import BinaryAUROC, MulticlassAUROC, MultilabelAUROC
|
||||
|
||||
from pyannote.audio.core.task import Problem, Task, get_dtype
|
||||
from pyannote.audio.core.io import Channel
|
||||
from pyannote.audio.core.task import Problem, Task, _CachedChannel, get_dtype
|
||||
from pyannote.audio.utils.random import create_rng_for_worker
|
||||
|
||||
Subsets = list(Subset.__args__)
|
||||
@@ -47,7 +48,20 @@ class SegmentationTask(Task):
|
||||
"""Methods common to most segmentation tasks"""
|
||||
|
||||
def get_file(self, file_id):
|
||||
return {"audio": self.prepared_data["audio-path"][file_id]}
|
||||
file = {"audio": self.prepared_data["audio-path"][file_id]}
|
||||
metadata = self.prepared_data["audio-metadata"][file_id]
|
||||
# Channels are cached as literal indices or named _CachedChannel modes.
|
||||
if "channel" in metadata.dtype.names:
|
||||
channel = int(metadata["channel"])
|
||||
if channel == _CachedChannel.RANDOM:
|
||||
file["channel"] = Channel.RANDOM
|
||||
elif channel == _CachedChannel.ALL:
|
||||
file["channel"] = Channel.ALL
|
||||
elif channel == _CachedChannel.DOWNMIX:
|
||||
file["channel"] = Channel.DOWNMIX
|
||||
elif channel >= 0:
|
||||
file["channel"] = channel
|
||||
return file
|
||||
|
||||
def default_metric(
|
||||
self,
|
||||
@@ -157,6 +171,8 @@ class SegmentationTask(Task):
|
||||
rng = create_rng_for_worker(self.model)
|
||||
|
||||
balance = getattr(self, "balance", None)
|
||||
if balance is not None and "channel" in balance:
|
||||
raise ValueError("Channel selection cannot be used for balancing.")
|
||||
if balance is None:
|
||||
chunks = self.train__iter__helper(rng)
|
||||
|
||||
|
||||
@@ -132,7 +132,8 @@ class Waveform:
|
||||
self._audio = Audio()
|
||||
|
||||
def __call__(self, file: ProtocolFile):
|
||||
waveform, _ = self._audio(file)
|
||||
# Preprocessors decode raw audio; consumers apply the channel policy.
|
||||
waveform, _ = self._audio({"audio": file["audio"], "channel": "all"})
|
||||
return waveform
|
||||
|
||||
|
||||
|
||||
@@ -81,10 +81,10 @@ def listen(audio_file: AudioFile, segment: Segment | None = None) -> None:
|
||||
return
|
||||
|
||||
if segment is None:
|
||||
waveform, sr = Audio(mono="downmix")(audio_file)
|
||||
waveform, sr = Audio()(audio_file)
|
||||
else:
|
||||
waveform, sr = Audio(mono="downmix").crop(audio_file, segment)
|
||||
return IPythonAudio(waveform.flatten(), rate=sr)
|
||||
waveform, sr = Audio().crop(audio_file, segment)
|
||||
return IPythonAudio(waveform.numpy(), rate=sr)
|
||||
|
||||
|
||||
def preview(
|
||||
@@ -157,7 +157,7 @@ def preview(
|
||||
temp_dir = tempfile.mkdtemp(prefix="pyannote-audio-preview")
|
||||
video_path = f"{temp_dir}/{uri}.{video_ext}"
|
||||
|
||||
audio = Audio(sample_rate=16000, mono="downmix")
|
||||
audio = Audio(sample_rate=16000)
|
||||
|
||||
if segment is None:
|
||||
duration = audio.get_duration(audio_file)
|
||||
|
||||
@@ -28,7 +28,7 @@ from pyannote.database.protocol import SpeakerVerificationProtocol
|
||||
|
||||
from pyannote.audio.core.io import Audio
|
||||
|
||||
get_duration = Audio(mono="downmix").get_duration
|
||||
get_duration = Audio().get_duration
|
||||
|
||||
|
||||
def check_protocol(protocol: Protocol) -> Protocol:
|
||||
|
||||
+95
-10
@@ -1,9 +1,12 @@
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from pyannote.core import Segment
|
||||
from torch import Tensor
|
||||
from torchcodec.decoders import AudioDecoder, AudioStreamMetadata
|
||||
|
||||
from pyannote.audio.core.io import Audio
|
||||
from pyannote.audio.core.io import Audio, Channel
|
||||
|
||||
|
||||
def test_audio_resample():
|
||||
@@ -11,7 +14,7 @@ def test_audio_resample():
|
||||
test_file = "tests/data/dev00.wav"
|
||||
metadata: AudioStreamMetadata = AudioDecoder(test_file).metadata
|
||||
old_sr = metadata.sample_rate
|
||||
loader = Audio(sample_rate=old_sr // 2, mono="downmix")
|
||||
loader = Audio(sample_rate=old_sr // 2)
|
||||
wav, sr = loader(test_file)
|
||||
assert isinstance(wav, Tensor)
|
||||
assert sr == old_sr // 2
|
||||
@@ -19,7 +22,7 @@ def test_audio_resample():
|
||||
|
||||
def test_basic_load_with_defaults():
|
||||
test_file = "tests/data/dev00.wav"
|
||||
loader = Audio(mono="downmix")
|
||||
loader = Audio()
|
||||
wav, _sr = loader(test_file)
|
||||
assert isinstance(wav, Tensor)
|
||||
|
||||
@@ -27,16 +30,98 @@ def test_basic_load_with_defaults():
|
||||
def test_correct_audio_channel():
|
||||
"When we specify an audio channel, it is chosen correctly"
|
||||
waveform = torch.rand(2, 16000 * 2)
|
||||
loader = Audio(mono="downmix")
|
||||
loader = Audio()
|
||||
wav, sr = loader({"waveform": waveform, "sample_rate": 16000, "channel": 1})
|
||||
assert torch.equal(wav, waveform[1:2])
|
||||
assert sr == 16000
|
||||
|
||||
|
||||
@pytest.mark.parametrize("crop", [False, True])
|
||||
@pytest.mark.parametrize("num_channels", [1, 3])
|
||||
def test_random_audio_channel(monkeypatch, crop, num_channels):
|
||||
waveform = torch.arange(num_channels).float().unsqueeze(1).repeat(1, 16000)
|
||||
file = {"waveform": waveform, "sample_rate": 16000, "channel": "random"}
|
||||
loader = Audio(sample_rate=8000)
|
||||
rng = random.Random(42)
|
||||
monkeypatch.setattr("pyannote.audio.core.io.random.randint", rng.randint)
|
||||
expected_rng = random.Random(42)
|
||||
for _ in range(5):
|
||||
channel = expected_rng.randint(0, num_channels - 1)
|
||||
selected = {**file, "channel": channel}
|
||||
if crop:
|
||||
actual, sr = loader.crop(file, Segment(0.25, 0.75))
|
||||
expected, _ = loader.crop(selected, Segment(0.25, 0.75))
|
||||
else:
|
||||
actual, sr = loader(file)
|
||||
expected, _ = loader(selected)
|
||||
assert sr == 8000
|
||||
assert actual.shape[0] == 1
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("crop", [False, True])
|
||||
@pytest.mark.parametrize("as_enum", [False, True])
|
||||
@pytest.mark.parametrize("num_channels", [1, 3])
|
||||
def test_all_audio_channels(crop, as_enum, num_channels):
|
||||
waveform = torch.randn(num_channels, 16000)
|
||||
file = {
|
||||
"waveform": waveform,
|
||||
"sample_rate": 16000,
|
||||
"channel": Channel.ALL if as_enum else "all",
|
||||
}
|
||||
loader = Audio(sample_rate=8000)
|
||||
if crop:
|
||||
actual, sr = loader.crop(file, Segment(0.25, 0.75))
|
||||
else:
|
||||
actual, sr = loader(file)
|
||||
|
||||
# Each output channel must match independently loading and resampling that
|
||||
# channel, preserving their original order.
|
||||
expected = []
|
||||
for channel in range(num_channels):
|
||||
selected = {**file, "channel": channel}
|
||||
if crop:
|
||||
single, _ = loader.crop(selected, Segment(0.25, 0.75))
|
||||
else:
|
||||
single, _ = loader(selected)
|
||||
expected.append(single)
|
||||
assert sr == 8000
|
||||
assert actual.shape[0] == num_channels
|
||||
torch.testing.assert_close(actual, torch.cat(expected))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("crop", [False, True])
|
||||
@pytest.mark.parametrize("as_enum", [False, True])
|
||||
@pytest.mark.parametrize("num_channels", [1, 3])
|
||||
def test_explicit_downmix(crop, as_enum, num_channels, monkeypatch):
|
||||
waveform = torch.randn(num_channels, 16000)
|
||||
file = {
|
||||
"waveform": waveform,
|
||||
"sample_rate": 16000,
|
||||
"channel": Channel.DOWNMIX if as_enum else "downmix",
|
||||
}
|
||||
averaged = {"waveform": waveform.mean(dim=0, keepdim=True), "sample_rate": 16000}
|
||||
loader = Audio(sample_rate=8000)
|
||||
|
||||
def unexpected_random(*args):
|
||||
pytest.fail("Explicit downmix must not randomly select a channel")
|
||||
|
||||
monkeypatch.setattr("pyannote.audio.core.io.random.randint", unexpected_random)
|
||||
if crop:
|
||||
actual, sr = loader.crop(file, Segment(0.25, 0.75))
|
||||
expected, _ = loader.crop(averaged, Segment(0.25, 0.75))
|
||||
else:
|
||||
actual, sr = loader(file)
|
||||
expected, _ = loader(averaged)
|
||||
assert sr == 8000
|
||||
assert actual.shape[0] == 1
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
def test_can_load_with_waveform():
|
||||
"We can load a raw waveform"
|
||||
waveform = torch.rand(2, 16000 * 2)
|
||||
loader = Audio(mono="downmix")
|
||||
loader = Audio()
|
||||
wav, sr = loader({"waveform": waveform, "sample_rate": 16000})
|
||||
assert isinstance(wav, Tensor)
|
||||
assert sr == 16000
|
||||
@@ -45,7 +130,7 @@ def test_can_load_with_waveform():
|
||||
def test_can_crop():
|
||||
"Cropping works when we give a Segment"
|
||||
test_file = "tests/data/dev00.wav"
|
||||
loader = Audio(mono="downmix")
|
||||
loader = Audio()
|
||||
segment = Segment(0.2, 0.7)
|
||||
wav, sr = loader.crop(test_file, segment)
|
||||
assert wav.shape[1] / sr == 0.5
|
||||
@@ -54,7 +139,7 @@ def test_can_crop():
|
||||
def test_can_crop_waveform():
|
||||
"Cropping works on raw waveforms"
|
||||
waveform = torch.rand(1, 16000 * 2)
|
||||
loader = Audio(mono="downmix")
|
||||
loader = Audio()
|
||||
segment = Segment(0.2, 0.7)
|
||||
wav, sr = loader.crop({"waveform": waveform, "sample_rate": 16000}, segment)
|
||||
assert isinstance(wav, Tensor)
|
||||
@@ -64,7 +149,7 @@ def test_can_crop_waveform():
|
||||
def test_can_crop_waveform_up_to_its_end():
|
||||
"Cropping a raw waveform up to its very last sample does not raise"
|
||||
waveform = torch.rand(1, 16000 * 2)
|
||||
loader = Audio(mono="downmix")
|
||||
loader = Audio()
|
||||
segment = Segment(1.5, 2.0)
|
||||
wav, _sr = loader.crop({"waveform": waveform, "sample_rate": 16000}, segment)
|
||||
assert torch.equal(wav, waveform[:, 24000:])
|
||||
@@ -73,7 +158,7 @@ def test_can_crop_waveform_up_to_its_end():
|
||||
# File Like Object Tests
|
||||
def test_can_load_from_file_like():
|
||||
"Load entire wav of file like"
|
||||
loader = Audio(mono="downmix")
|
||||
loader = Audio()
|
||||
|
||||
with open("tests/data/dev00.wav", "rb") as f:
|
||||
wav, sr = loader(f)
|
||||
@@ -84,7 +169,7 @@ def test_can_load_from_file_like():
|
||||
|
||||
def test_can_crop_from_file_like():
|
||||
"Load cropped sections from file like objects"
|
||||
loader = Audio(mono="downmix")
|
||||
loader = Audio()
|
||||
|
||||
with open("tests/data/dev00.wav", "rb") as f:
|
||||
segment = Segment(0.2, 0.7)
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
import random
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from pyannote.core import Annotation, Segment, Timeline
|
||||
from pyannote.database.protocol import SpeakerDiarizationProtocol
|
||||
|
||||
from pyannote.audio.core import io
|
||||
from pyannote.audio.core.io import Channel
|
||||
from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel
|
||||
from pyannote.audio.tasks import SpeakerDiarization
|
||||
|
||||
|
||||
class ChannelProtocol(SpeakerDiarizationProtocol):
|
||||
name = "Test.SpeakerDiarization.Channels"
|
||||
|
||||
def __init__(self, audio_path, channels, validation_channels=None):
|
||||
super().__init__()
|
||||
self.audio_path = audio_path
|
||||
self.channels = channels
|
||||
self.validation_channels = (
|
||||
channels if validation_channels is None else validation_channels
|
||||
)
|
||||
|
||||
def files(self, subset):
|
||||
channels = self.channels if subset == "train" else self.validation_channels
|
||||
for index, channel in enumerate(channels):
|
||||
annotation = Annotation()
|
||||
annotation[Segment(0, 1)] = "speaker"
|
||||
file = {
|
||||
"uri": f"{subset}-{index}",
|
||||
"database": f"Database{index}",
|
||||
"subset": subset,
|
||||
"scope": "file",
|
||||
"audio": self.audio_path,
|
||||
"annotation": annotation,
|
||||
"annotated": Timeline([Segment(0, 1)]),
|
||||
}
|
||||
if channel is not None:
|
||||
file["channel"] = channel
|
||||
yield file
|
||||
|
||||
def train_iter(self):
|
||||
yield from self.files("train")
|
||||
|
||||
def development_iter(self):
|
||||
yield from self.files("development")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"channels",
|
||||
[
|
||||
(0,),
|
||||
(2,),
|
||||
(None,),
|
||||
(None, 2, 0),
|
||||
("random",),
|
||||
("all",),
|
||||
("downmix",),
|
||||
(Channel.RANDOM, Channel.ALL, Channel.DOWNMIX),
|
||||
(0, "random", None, 2, "all", "downmix"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("stage", ["train", "development"])
|
||||
def test_cached_channel_selection(tmp_path, monkeypatch, channels, stage):
|
||||
# Mock only decoding: exercise the real crop, channel selection, and downmix.
|
||||
# Distinct constant channels make selected and averaged audio distinguishable.
|
||||
waveform = torch.tensor([0.125, 0.25, 0.75]).unsqueeze(1).repeat(1, 16000)
|
||||
audio_path = tmp_path / "multichannel.wav"
|
||||
audio_path.touch()
|
||||
|
||||
class Decoder:
|
||||
def __init__(self, path):
|
||||
assert str(path) == str(audio_path)
|
||||
self.metadata = SimpleNamespace(
|
||||
sample_rate=16000, duration_seconds_from_header=1.0
|
||||
)
|
||||
|
||||
def get_samples_played_in_range(self, start, end):
|
||||
return SimpleNamespace(
|
||||
data=waveform[:, round(start * 16000) : round(end * 16000)],
|
||||
sample_rate=16000,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(io, "AudioDecoder", Decoder, raising=False)
|
||||
monkeypatch.setattr(io, "TORCHCODEC_AVAILABLE", True)
|
||||
task = SpeakerDiarization(
|
||||
ChannelProtocol(audio_path, channels),
|
||||
cache=tmp_path / "task.npz",
|
||||
duration=1.0,
|
||||
max_speakers_per_chunk=2,
|
||||
num_workers=0,
|
||||
)
|
||||
task.prepare_data()
|
||||
task.setup()
|
||||
assert "channel" in task.prepared_data["audio-metadata"].dtype.names
|
||||
assert "channel" not in task.prepared_data["metadata-values"]
|
||||
task.model = SimpleSegmentationModel()
|
||||
|
||||
for index, channel in enumerate(channels):
|
||||
file_id = index + (len(channels) if stage == "development" else 0)
|
||||
file = task.get_file(file_id)
|
||||
if channel is None:
|
||||
assert "channel" not in file
|
||||
else:
|
||||
assert file["channel"] == channel
|
||||
|
||||
selections = []
|
||||
|
||||
def select_channel(low, high):
|
||||
assert (low, high) == (0, 2)
|
||||
selected = (len(selections) + 1) % 3
|
||||
selections.append(selected)
|
||||
return selected
|
||||
|
||||
monkeypatch.setattr(io.random, "randint", select_channel)
|
||||
if stage == "train":
|
||||
# Select each source database in a compound protocol deterministically.
|
||||
samples = task.train__iter__helper(
|
||||
random.Random(0), database=f"Database{index}"
|
||||
)
|
||||
sample = next(samples)
|
||||
else:
|
||||
sample = task.val__getitem__(index)
|
||||
|
||||
if channel == "random":
|
||||
torch.testing.assert_close(sample["X"], waveform[1:2])
|
||||
second = next(samples) if stage == "train" else task.val__getitem__(index)
|
||||
torch.testing.assert_close(second["X"], waveform[2:3])
|
||||
assert selections == [1, 2]
|
||||
continue
|
||||
|
||||
assert not selections
|
||||
if channel == "all":
|
||||
expected = waveform
|
||||
elif channel is None or channel == "downmix":
|
||||
expected = waveform.mean(dim=0, keepdim=True)
|
||||
else:
|
||||
expected = waveform[channel : channel + 1]
|
||||
torch.testing.assert_close(sample["X"], expected)
|
||||
torch.testing.assert_close(
|
||||
task.collate_X([sample, sample]), torch.stack([expected, expected])
|
||||
)
|
||||
if stage == "development":
|
||||
torch.testing.assert_close(task.val__getitem__(index)["X"], expected)
|
||||
assert not selections
|
||||
|
||||
|
||||
def make_task(tmp_path, validation_channels=(0,)):
|
||||
return SpeakerDiarization(
|
||||
ChannelProtocol(tmp_path / "audio.wav", ("random",), validation_channels),
|
||||
cache=tmp_path / "task.npz",
|
||||
duration=1.0,
|
||||
max_speakers_per_chunk=2,
|
||||
num_workers=0,
|
||||
)
|
||||
|
||||
|
||||
def test_channel_balancing_rejected(tmp_path):
|
||||
task = make_task(tmp_path)
|
||||
task.prepare_data()
|
||||
task.setup()
|
||||
task.model = SimpleSegmentationModel()
|
||||
task.balance = ["channel"]
|
||||
with pytest.raises(
|
||||
ValueError, match="Channel selection cannot be used for balancing"
|
||||
):
|
||||
next(task.train__iter__())
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Channel policies behave identically with raw, preloaded, and cached audio."""
|
||||
|
||||
import random
|
||||
import wave
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from pyannote.core import Annotation, Segment
|
||||
from pyannote.database.protocol import SpeakerVerificationProtocol
|
||||
from pyannote.database.protocol.protocol import ProtocolFile
|
||||
|
||||
from pyannote.audio.core.io import Audio, Channel
|
||||
from pyannote.audio.core.pipeline import Pipeline
|
||||
from pyannote.audio.tasks.embedding.mixins import (
|
||||
SupervisedRepresentationLearningTaskMixin,
|
||||
)
|
||||
from pyannote.audio.utils.preprocessors import Waveform
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def multichannel_file(tmp_path):
|
||||
samples = np.tile(np.array([4096, 8192, 24576], dtype="<i2"), (16000, 1))
|
||||
path = tmp_path / "channels.wav"
|
||||
with wave.open(str(path), "wb") as output:
|
||||
output.setnchannels(3)
|
||||
output.setsampwidth(2)
|
||||
output.setframerate(16000)
|
||||
output.writeframes(samples.tobytes())
|
||||
return {"audio": path}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("channel", [None, 0, 2, *Channel, "all", "downmix", "random"])
|
||||
def test_preloading_preserves_policy(multichannel_file, channel, monkeypatch):
|
||||
file = {**multichannel_file, "channel": channel}
|
||||
pipeline = Pipeline()
|
||||
raw = pipeline.prepare_one(dict(file), preload=False)
|
||||
preloaded = pipeline.prepare_one(dict(file), preload=True)
|
||||
assert preloaded["waveform"].shape == (3, 16000)
|
||||
assert preloaded["channel"] == channel
|
||||
loader = Audio(sample_rate=8000)
|
||||
for cropped in (False, True):
|
||||
# Identical random state must give identical selections with preloading.
|
||||
outputs = []
|
||||
for source in (raw, preloaded):
|
||||
monkeypatch.setattr(
|
||||
"pyannote.audio.core.io.random.randint", random.Random(7).randint
|
||||
)
|
||||
result = (
|
||||
loader.crop(source, Segment(0.25, 0.75)) if cropped else loader(source)
|
||||
)
|
||||
outputs.append(result[0])
|
||||
torch.testing.assert_close(*outputs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("channel", [2, Channel.ALL, Channel.DOWNMIX])
|
||||
def test_waveform_preprocessor_preserves_raw_channels(multichannel_file, channel):
|
||||
file = ProtocolFile(
|
||||
{**multichannel_file, "channel": channel, "sample_rate": 16000},
|
||||
lazy={"waveform": Waveform()},
|
||||
)
|
||||
assert file["waveform"].shape == (3, 16000)
|
||||
actual, _ = Audio()(file)
|
||||
expected, _ = Audio()({**multichannel_file, "channel": channel})
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("crop", [False, True])
|
||||
@pytest.mark.parametrize("explicit_none", [False, True])
|
||||
def test_missing_channel_defaults_to_downmix(crop, explicit_none):
|
||||
waveform = torch.rand(3, 16000)
|
||||
file = {"waveform": waveform, "sample_rate": 16000}
|
||||
if explicit_none:
|
||||
file["channel"] = None
|
||||
loader = Audio()
|
||||
actual, _ = loader.crop(file, Segment(0.25, 0.75)) if crop else loader(file)
|
||||
expected = waveform.mean(dim=0, keepdim=True)
|
||||
if crop:
|
||||
expected = expected[:, 4000:12000]
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("channel", [-1, -2, True, 1.5, "invalid", 3])
|
||||
def test_invalid_channel_rejected(channel):
|
||||
with pytest.raises(ValueError, match="channel"):
|
||||
Audio()(
|
||||
{"waveform": torch.ones(3, 16), "sample_rate": 16000, "channel": channel}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("channel", [0, Channel.RANDOM, Channel.ALL, Channel.DOWNMIX])
|
||||
def test_embedding_training_keeps_channel(multichannel_file, channel):
|
||||
annotation = Annotation()
|
||||
annotation[Segment(0, 1)] = "speaker"
|
||||
file = {
|
||||
**multichannel_file,
|
||||
"uri": "test",
|
||||
"channel": channel,
|
||||
"annotation": annotation,
|
||||
}
|
||||
task = SimpleNamespace(
|
||||
protocol=SimpleNamespace(name="test", train=lambda: iter([file])),
|
||||
min_duration=0.5,
|
||||
duration=1,
|
||||
)
|
||||
SupervisedRepresentationLearningTaskMixin.setup(task)
|
||||
assert task._train["speaker"][0]["channel"] == channel
|
||||
|
||||
|
||||
@pytest.mark.parametrize("channel", ["random", Channel.RANDOM])
|
||||
@pytest.mark.parametrize("duration", [0.5, 1.0])
|
||||
def test_embedding_validation_supports_random(
|
||||
multichannel_file, channel, duration, monkeypatch
|
||||
):
|
||||
file = {**multichannel_file, "channel": channel}
|
||||
task = SimpleNamespace(
|
||||
protocol=SpeakerVerificationProtocol(),
|
||||
prepared_data={
|
||||
"validation": [{"file1": file, "file2": file, "reference": True}]
|
||||
},
|
||||
model=SimpleNamespace(audio=Audio(sample_rate=16000)),
|
||||
duration=duration,
|
||||
)
|
||||
selections = iter([1, 2, 2, 0])
|
||||
calls = []
|
||||
|
||||
def select_channel(low, high):
|
||||
assert (low, high) == (0, 2)
|
||||
selected = next(selections)
|
||||
calls.append(selected)
|
||||
return selected
|
||||
|
||||
monkeypatch.setattr("pyannote.audio.core.io.random.randint", select_channel)
|
||||
for expected_values in [(0.25, 0.75), (0.75, 0.125)]:
|
||||
sample = SupervisedRepresentationLearningTaskMixin.val__getitem__(task, 0)
|
||||
for index, value in enumerate(expected_values, start=1):
|
||||
torch.testing.assert_close(
|
||||
sample[f"X{index}"], torch.full((1, int(duration * 16000)), value)
|
||||
)
|
||||
assert sample["y"] is True
|
||||
assert calls == [1, 2, 2, 0]
|
||||
@@ -0,0 +1,93 @@
|
||||
import sys
|
||||
import wave
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from pyannote.audio.core.io import Audio, Channel
|
||||
from pyannote.audio.pipelines.pyannoteai.local import Local
|
||||
from pyannote.audio.pipelines.pyannoteai.sdk import SDK
|
||||
|
||||
|
||||
@pytest.fixture(params=["local", "sdk"])
|
||||
def wrapper(request, monkeypatch):
|
||||
backend = Mock()
|
||||
if request.param == "local":
|
||||
backend.diarize.return_value = {"diarization": [], "exclusive_diarization": []}
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"pyannoteai.local",
|
||||
SimpleNamespace(Pipeline=lambda token: backend),
|
||||
)
|
||||
pipeline = Local(token="test")
|
||||
payload = backend.diarize
|
||||
else:
|
||||
backend.upload.return_value = "media://test"
|
||||
backend.diarize.return_value = "test-job"
|
||||
backend.retrieve.return_value = {
|
||||
"output": {"diarization": [], "exclusiveDiarization": []}
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
"pyannote.audio.pipelines.pyannoteai.sdk.Client", lambda token: backend
|
||||
)
|
||||
pipeline = SDK(token="test")
|
||||
payload = backend.upload
|
||||
return pipeline, backend, payload
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def audio_source(tmp_path):
|
||||
waveform = torch.tensor([0.125, 0.25, 0.75]).unsqueeze(1).repeat(1, 16000)
|
||||
path = tmp_path / "channels.wav"
|
||||
with wave.open(str(path), "wb") as output:
|
||||
output.setnchannels(3)
|
||||
output.setsampwidth(2)
|
||||
output.setframerate(16000)
|
||||
output.writeframes((waveform.T.numpy() * 32768).astype("<i2").tobytes())
|
||||
return path, waveform
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["path", "waveform", "preloaded"])
|
||||
@pytest.mark.parametrize("channel", [None, 0, 2, Channel.DOWNMIX, Channel.RANDOM])
|
||||
def test_wrappers_apply_selection(wrapper, audio_source, source, channel, monkeypatch):
|
||||
pipeline, _, payload = wrapper
|
||||
path, waveform = audio_source
|
||||
file = {"audio": path, "channel": channel}
|
||||
if source == "waveform":
|
||||
# Audio must prefer this waveform, rather than reread the path.
|
||||
file.update(audio="missing.wav", waveform=waveform, sample_rate=16000)
|
||||
monkeypatch.setattr("pyannote.audio.core.io.random.randint", lambda low, high: 2)
|
||||
pipeline(file, preload=source == "preloaded", num_speakers=2)
|
||||
processed = payload.call_args.args[0]
|
||||
expected, _ = Audio()(
|
||||
{"waveform": waveform, "sample_rate": 16000, "channel": channel}
|
||||
)
|
||||
assert set(processed) == {"waveform", "sample_rate"}
|
||||
np.testing.assert_allclose(processed["waveform"], expected.numpy())
|
||||
assert processed["sample_rate"] == 16000
|
||||
assert processed["waveform"].shape == (1, 16000)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["path", "waveform", "preloaded"])
|
||||
def test_wrappers_reject_multichannel_all(wrapper, audio_source, source):
|
||||
pipeline, backend, _ = wrapper
|
||||
path, waveform = audio_source
|
||||
file = {"audio": path, "channel": Channel.ALL}
|
||||
if source == "waveform":
|
||||
file.update(waveform=waveform, sample_rate=16000)
|
||||
with pytest.raises(ValueError, match="require mono audio"):
|
||||
pipeline(file, preload=source == "preloaded")
|
||||
backend.diarize.assert_not_called()
|
||||
backend.upload.assert_not_called()
|
||||
|
||||
|
||||
def test_wrappers_accept_all_for_mono(wrapper):
|
||||
pipeline, _, payload = wrapper
|
||||
waveform = torch.ones(1, 16000)
|
||||
pipeline({"waveform": waveform, "sample_rate": 16000, "channel": Channel.ALL})
|
||||
np.testing.assert_array_equal(
|
||||
payload.call_args.args[0]["waveform"], waveform.numpy()
|
||||
)
|
||||
@@ -211,7 +211,7 @@
|
||||
"source": [
|
||||
"from pyannote.audio import Audio\n",
|
||||
"from IPython.display import Audio as IPythonAudio\n",
|
||||
"waveform, sr = Audio(mono=\"downmix\").crop(DEMO_FILE, EXCERPT)\n",
|
||||
"waveform, sr = Audio().crop(DEMO_FILE, EXCERPT)\n",
|
||||
"IPythonAudio(waveform.flatten(), rate=sr)"
|
||||
]
|
||||
},
|
||||
@@ -251,7 +251,7 @@
|
||||
"notebook.reset()\n",
|
||||
"\n",
|
||||
"# load audio waveform and play it\n",
|
||||
"waveform, sample_rate = Audio(mono=\"downmix\")(OWN_FILE)\n",
|
||||
"waveform, sample_rate = Audio()(OWN_FILE)\n",
|
||||
"IPythonAudio(data=waveform.squeeze(), rate=sample_rate, autoplay=True)"
|
||||
]
|
||||
},
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
"\n",
|
||||
"from pyannote.audio import Audio\n",
|
||||
"from pyannote.core import Segment\n",
|
||||
"audio = Audio(sample_rate=16000, mono=\"downmix\")\n",
|
||||
"audio = Audio(sample_rate=16000)\n",
|
||||
"\n",
|
||||
"# extract embedding for a speaker speaking between t=3s and t=6s\n",
|
||||
"speaker1 = Segment(3., 6.)\n",
|
||||
|
||||
Reference in new issue
Block a user