BREAKING: perform channel selection per-file according to 'channel' key

This commit is contained in:
Hervé BREDIN authored and GitHub committed 2026-11-17 08:51:15 +01:00
1 parent 0d6f291328
commit 47fa6ac0dd
24 files changed
+696 -86

No files matched your search

+3
View File
@@ -2,8 +2,11 @@
## next
- BREAKING(io): rename `Audio.downmix_and_resample` to `Audio.select_channels_and_resample`
- BREAKING(io): remove `Audio(mono=...)`; use the file's `channel` policy instead.
- BREAKING(cli): remove gap-filling optimization (`--optimize`) and `OptimizedMinDurationOff` outputs from the `benchmark` command
- BREAKING: require Python 3.11 or newer; Python 3.10 is no longer supported
- feat(task): support `channel: {int | downmix | random | all}` for audio loading (requires `pyannote.database` >= 6.3.0)
- fix(model): restore `fbank_only` mode in WeSpeaker embedding models
- improve(model): compute WeSpeaker fbank features on MPS instead of falling back to CPU [@Isalia20](https://github.com/Isalia20)
- fix(task): avoid training crashes with `spawn`/`forkserver` workers, including
+1 -1
View File
@@ -12,7 +12,7 @@
" preprocessors={\"audio\": FileFinder()})\n",
"\n",
"from pyannote.audio.core.io import Audio\n",
"audio = Audio(sample_rate=16000, mono=\"downmix\")\n",
"audio = Audio(sample_rate=16000)\n",
"file = next(protocol.test())\n",
"\n",
"from pyannote.core import Segment\n",
+1 -1
View File
@@ -17,7 +17,7 @@ dependencies = [
"opentelemetry-sdk>=1.34.0",
"opentelemetry-exporter-otlp>=1.34.0",
"pyannote-core>=6.1.0",
"pyannote-database>=6.1.1",
"pyannote-database>=6.3.0",
"pyannote-metrics>=4.2.0",
"pyannote-pipeline>=4.1.0",
"pytorch-metric-learning>=2.8.1",
+82 -25
View File
@@ -31,7 +31,9 @@ pyannote.audio relies on torchcodec for reading and torchaudio for resampling.
import random
import warnings
from collections.abc import Mapping
from enum import StrEnum
from io import IOBase
from numbers import Integral
from pathlib import Path
import torch.nn.functional as F
@@ -56,6 +58,50 @@ except Exception as e:
AudioStreamMetadata = None
class Channel(StrEnum):
"""Channel selection options"""
RANDOM = "random"
ALL = "all"
DOWNMIX = "downmix"
@classmethod
def normalize(cls, channel) -> "int | Channel":
"""Normalize channel selection
Parameters
----------
channel : int, str, Channel, or None
Zero-based channel index or channel selection option.
Supported options are "random", "all", and "downmix".
None defaults to "downmix".
Returns
-------
channel : int or Channel
Non-negative Python integer or Channel enum member.
Raises
------
ValueError
If channel is not a non-negative integer, a supported option, or None.
"""
if channel is None:
return cls.DOWNMIX
if isinstance(channel, Integral) and not isinstance(channel, bool):
if channel >= 0:
return int(channel)
elif isinstance(channel, str):
try:
return cls(channel)
except ValueError:
pass
raise ValueError(
'channel must be a non-negative integer, "random", "all", or "downmix".'
)
AudioFile = str | Path | IOBase | Mapping
AudioFileDocString = """
@@ -68,6 +114,9 @@ Audio files can be provided to the Audio class using different types:
For last two options, an additional "channel" key can be provided as a zero-indexed
integer to load a specific channel: {"audio": "stereo.wav", "channel": 0}
or "random" to select a channel randomly on each load or crop. Use "all" to
preserve all channels in their original order. Use "downmix" (the default when
channel is omitted or None) to average all channels.
"""
@@ -114,14 +163,14 @@ class Audio:
----------
sample_rate: int, optional
Target sampling rate. Defaults to using native sampling rate.
mono : {'random', 'downmix'}, optional
In case of multi-channel audio, convert to single-channel audio
using one of the following strategies: select one channel at
'random' or 'downmix' by averaging all channels.
Notes
-----
Channel selection is controlled by the file dictionary, defaulting to "downmix".
Usage
-----
>>> audio = Audio(sample_rate=16000, mono='downmix')
>>> audio = Audio(sample_rate=16000)
>>> waveform, sample_rate = audio({"audio": "/path/to/audio.wav"})
>>> assert sample_rate == 16000
>>> sample_rate = 44100
@@ -215,15 +264,17 @@ class Audio:
return file
def __init__(self, sample_rate: int | None = None, mono=None):
def __init__(self, sample_rate: int | None = None):
super().__init__()
self.sample_rate = sample_rate
self.mono = mono
def downmix_and_resample(
self, waveform: Tensor, sample_rate: int, channel: int | None = None
def select_channels_and_resample(
self,
waveform: Tensor,
sample_rate: int,
channel: int | str | Channel | None = None,
) -> tuple[Tensor, int]:
"""Downmix and resample
"""Apply the channel policy and resample
Parameters
----------
@@ -231,28 +282,32 @@ class Audio:
Waveform.
sample_rate : int
Sample rate.
channel : int, optional
Channel to use.
channel : int, "random", "all", or "downmix", optional
Zero-based channel to use, "random" to select one on each call, or
"all" to preserve every channel. "downmix" averages all channels.
Defaults to "downmix" when omitted or None.
Returns
-------
waveform : (channel, time) Tensor
Remixed and resampled waveform
Waveform after applying the channel policy and resampling
sample_rate : int
New sample rate
"""
# select channel if requested
if channel is not None:
waveform = waveform[channel : channel + 1]
# downmix to mono
channel = Channel.normalize(channel)
num_channels = waveform.shape[0]
if num_channels > 1:
if self.mono == "random":
if num_channels < 1:
raise ValueError("Audio must contain at least one channel.")
if channel == Channel.RANDOM:
channel = random.randint(0, num_channels - 1)
if isinstance(channel, int):
if channel >= num_channels:
raise ValueError(
f"Requested channel {channel}, but audio has {num_channels} channels."
)
waveform = waveform[channel : channel + 1]
elif self.mono == "downmix":
elif channel == Channel.DOWNMIX and num_channels > 1:
waveform = waveform.mean(dim=0, keepdim=True)
# resample
@@ -327,7 +382,9 @@ class Audio:
waveform = file["waveform"]
sample_rate = file["sample_rate"]
return self.downmix_and_resample(waveform, sample_rate, channel=channel)
return self.select_channels_and_resample(
waveform, sample_rate, channel=channel
)
if not TORCHCODEC_AVAILABLE:
raise RuntimeError(
@@ -346,7 +403,7 @@ class Audio:
if isinstance(file["audio"], IOBase):
file["audio"].seek(0)
return self.downmix_and_resample(waveform, sample_rate, channel=channel)
return self.select_channels_and_resample(waveform, sample_rate, channel=channel)
def crop(
self,
@@ -409,7 +466,7 @@ class Audio:
data = waveform[:, start_sample:end_sample]
data = F.pad(data, (pad_start, pad_end))
return self.downmix_and_resample(data, sample_rate, channel=channel)
return self.select_channels_and_resample(data, sample_rate, channel=channel)
if not TORCHCODEC_AVAILABLE:
raise RuntimeError(
@@ -479,4 +536,4 @@ class Audio:
pad_end += 1
data = F.pad(data, (pad_start, pad_end))
return self.downmix_and_resample(data, sample_rate, channel=channel)
return self.select_channels_and_resample(data, sample_rate, channel=channel)
+3 -4
View File
@@ -88,14 +88,13 @@ class Model(lightning.LightningModule):
):
super().__init__()
assert num_channels == 1, (
"Only mono audio is supported for now (num_channels = 1)"
)
if not isinstance(num_channels, int) or num_channels < 1:
raise ValueError("num_channels must be a positive integer.")
self.save_hyperparameters("sample_rate", "num_channels")
self.task = task
self.audio = Audio(sample_rate=self.hparams.sample_rate, mono="downmix")
self.audio = Audio(sample_rate=self.hparams.sample_rate)
@property
def task(self) -> Task:
+3 -6
View File
@@ -482,12 +482,9 @@ class Pipeline(_Pipeline):
"Cannot preload audio: `waveform` key is already available or will be via a preprocessor."
)
# load waveform in memory (and keep track of its original sample rate)
file["waveform"], file["sample_rate"] = Audio()(file)
# the above line already took care of channel selection,
# therefore we remove the `channel` key from the file
file.pop("channel", None)
# load waveform in memory and keep track of its original sample rate.
# keep all channels for now: channel selection is done later at load/crop time.
file["waveform"], file["sample_rate"] = Audio()({**file, "channel": "all"})
return file
+30 -5
View File
@@ -32,7 +32,7 @@ import warnings
from collections import defaultdict
from collections.abc import Sequence
from dataclasses import dataclass
from enum import Enum
from enum import Enum, IntEnum
from functools import cached_property, partial
from numbers import Number
from pathlib import Path
@@ -50,12 +50,24 @@ from torch_audiomentations import Identity
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
from torchmetrics import Metric, MetricCollection
from pyannote.audio.core.io import Channel
from pyannote.audio.utils.loss import binary_cross_entropy, nll_loss
from pyannote.audio.utils.protocol import check_protocol
Subsets = list(Subset.__args__)
Scopes = list(Scope.__args__)
MISSING_METADATA = -1
class _CachedChannel(IntEnum):
"""Private integer encoding for channel selection in task caches."""
UNSPECIFIED = MISSING_METADATA
RANDOM = -2
ALL = -3
DOWNMIX = -4
# Type of machine learning problem
class Problem(Enum):
@@ -332,7 +344,7 @@ class Task(lightning.LightningDataModule):
prepared_data = {
'protocol': name of the protocol
'audio-path': array of N paths to audio
'audio-metadata': array of N audio infos such as audio subset, scope and database
'audio-metadata': array of N audio infos such as audio channel, subset, scope and database.
'audio-annotated': array of N annotated duration (usually equals file duration but might be shorter if file is not fully annotated)
'annotations-regions': array of M annotated regions
'audio-regions-ids': array of N start/end indices of annotated regions
@@ -402,6 +414,16 @@ class Task(lightning.LightningDataModule):
# keep track of label scope (file, database, or global)
metadatum["scope"] = Scopes.index(file["scope"])
# Channel selection is a loading option, not categorical metadata.
metadatum["channel"] = _CachedChannel.UNSPECIFIED
if "channel" in file:
channel = Channel.normalize(file["channel"])
metadatum["channel"] = (
_CachedChannel[channel.name]
if isinstance(channel, Channel)
else channel
)
remaining_metadata_keys = set(file) - {
"uri",
"database",
@@ -409,6 +431,7 @@ class Task(lightning.LightningDataModule):
"audio",
"scope",
"classes",
"channel",
"annotation",
"annotated",
}
@@ -525,14 +548,16 @@ class Task(lightning.LightningDataModule):
# append segment IDs (start and end time)
audio_segments_ids.append((_segments_id, segments_id))
# since not all metadata keys are present in all files, fallback to -1 when a key is missing
# Register channel separately without collecting its possible values.
metadata_keys = [*metadata_unique_values, "channel"]
# Not all metadata keys are present in every file.
metadata = [
tuple(metadatum.get(key, -1) for key in metadata_unique_values)
tuple(metadatum.get(key, MISSING_METADATA) for key in metadata_keys)
for metadatum in metadata
]
metadata_dtype = [
(key, get_dtype(max(m[i] for m in metadata)))
for i, key in enumerate(metadata_unique_values)
for i, key in enumerate(metadata_keys)
]
# turn list of annotated regions into a single numpy array
@@ -20,3 +20,32 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from pyannote.audio.core.io import Audio, AudioFile
def _prepare_audio(file: AudioFile) -> dict:
"""Apply channel selection before handing audio to a mono-only backend.
Parameters
----------
file : AudioFile
Audio file with an optional channel selection policy.
Returns
-------
audio : dict
Selected mono waveform as a NumPy array and its sample rate.
Raises
------
ValueError
If the selected waveform contains more than one channel.
"""
waveform, sample_rate = Audio()(file)
if waveform.shape[0] != 1:
raise ValueError(
'pyannoteAI backends require mono audio. Use channel="downmix", '
'"random", or a fixed channel index instead of "all" for multi-channel audio.'
)
return {"waveform": waveform.numpy(force=True), "sample_rate": sample_rate}
@@ -28,6 +28,7 @@ from pyannote.audio import Pipeline
from pyannote.audio.core.io import AudioFile
from ..speaker_diarization import DiarizeOutput
from . import _prepare_audio
class Local(Pipeline):
@@ -89,34 +90,21 @@ class Local(Pipeline):
-------
output : DiarizeOutput
DiarizeOutput object containing both regular and exclusive speaker diarization results.
Raises
------
ValueError
If channel selection leaves multiple channels. The backend requires mono audio.
"""
# if file provides "audio" path
if "audio" in file:
predictions = self._pipeline.diarize(
file["audio"],
_prepare_audio(file),
num_speakers=num_speakers,
min_speakers=min_speakers,
max_speakers=max_speakers,
**kwargs,
)
# if file provides "waveform", make sure it is numpy (and not torch) array
elif "waveform" in file:
waveform = file["waveform"]
if hasattr(waveform, "numpy"):
waveform = waveform.numpy(force=True)
predictions = self._pipeline.diarize(
{"waveform": waveform, "sample_rate": file["sample_rate"]},
num_speakers=num_speakers,
min_speakers=min_speakers,
max_speakers=max_speakers,
**kwargs,
)
else:
raise ValueError("AudioFile must provide either 'audio' or 'waveform' key")
speaker_diarization: Annotation = self._deserialize(predictions["diarization"])
exclusive_speaker_diarization: Annotation = self._deserialize(
predictions["exclusive_diarization"]
@@ -29,6 +29,8 @@ from pyannote.audio import Pipeline
from pyannote.audio.core.io import AudioFile
from pyannote.audio.pipelines.speaker_diarization import DiarizeOutput
from . import _prepare_audio
class SDK(Pipeline):
"""Wrapper around official pyannoteAI API client
@@ -97,6 +99,8 @@ class SDK(Pipeline):
Raises
------
ValueError
If channel selection leaves multiple channels. The backend requires mono audio.
PyannoteAIFailedJob
If the job failed
PyannoteAICanceledJob
@@ -106,7 +110,7 @@ class SDK(Pipeline):
"""
# upload file to pyannoteAI cloud API
media_url: str = self._client.upload(file)
media_url: str = self._client.upload(_prepare_audio(file))
# initiate diarization job
job_id = self._client.diarize(
@@ -268,7 +268,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
self._embedding = PretrainedSpeakerEmbedding(
self.embedding, token=token, cache_dir=cache_dir
)
self._audio = Audio(sample_rate=self._embedding.sample_rate, mono="downmix")
self._audio = Audio(sample_rate=self._embedding.sample_rate)
metric = self._embedding.metric
try:
@@ -180,7 +180,7 @@ class SpeechSeparation(SpeakerDiarizationMixin, Pipeline):
self._embedding = PretrainedSpeakerEmbedding(
self.embedding, token=token, cache_dir=cache_dir
)
self._audio = Audio(sample_rate=self._embedding.sample_rate, mono="downmix")
self._audio = Audio(sample_rate=self._embedding.sample_rate)
metric = self._embedding.metric
try:
+1 -1
View File
@@ -61,7 +61,7 @@ def oracle_segmentation(
"""
if "duration" not in file:
duration = Audio(mono="downmix").get_duration(file)
duration = Audio().get_duration(file)
else:
duration: float = file["duration"]
reference: Annotation = file["annotation"]
@@ -36,6 +36,7 @@ from torchmetrics import Metric
from torchmetrics.classification import BinaryAUROC
from tqdm import tqdm
from pyannote.audio.core.io import Channel
from pyannote.audio.core.task import Problem, Resolution, Specifications
from pyannote.audio.torchmetrics.classification import EqualErrorRate
from pyannote.audio.utils.random import create_rng_for_worker
@@ -107,6 +108,7 @@ class SupervisedRepresentationLearningTaskMixin:
{
"uri": f["uri"],
"audio": f["audio"],
"channel": Channel.normalize(f.get("channel")),
"duration": duration,
"speech_turns": speech_turns,
}
@@ -36,7 +36,8 @@ from torch.utils.data._utils.collate import default_collate
from torchmetrics import Metric
from torchmetrics.classification import BinaryAUROC, MulticlassAUROC, MultilabelAUROC
from pyannote.audio.core.task import Problem, Task, get_dtype
from pyannote.audio.core.io import Channel
from pyannote.audio.core.task import Problem, Task, _CachedChannel, get_dtype
from pyannote.audio.utils.random import create_rng_for_worker
Subsets = list(Subset.__args__)
@@ -47,7 +48,20 @@ class SegmentationTask(Task):
"""Methods common to most segmentation tasks"""
def get_file(self, file_id):
return {"audio": self.prepared_data["audio-path"][file_id]}
file = {"audio": self.prepared_data["audio-path"][file_id]}
metadata = self.prepared_data["audio-metadata"][file_id]
# Channels are cached as literal indices or named _CachedChannel modes.
if "channel" in metadata.dtype.names:
channel = int(metadata["channel"])
if channel == _CachedChannel.RANDOM:
file["channel"] = Channel.RANDOM
elif channel == _CachedChannel.ALL:
file["channel"] = Channel.ALL
elif channel == _CachedChannel.DOWNMIX:
file["channel"] = Channel.DOWNMIX
elif channel >= 0:
file["channel"] = channel
return file
def default_metric(
self,
@@ -157,6 +171,8 @@ class SegmentationTask(Task):
rng = create_rng_for_worker(self.model)
balance = getattr(self, "balance", None)
if balance is not None and "channel" in balance:
raise ValueError("Channel selection cannot be used for balancing.")
if balance is None:
chunks = self.train__iter__helper(rng)
+2 -1
View File
@@ -132,7 +132,8 @@ class Waveform:
self._audio = Audio()
def __call__(self, file: ProtocolFile):
waveform, _ = self._audio(file)
# Preprocessors decode raw audio; consumers apply the channel policy.
waveform, _ = self._audio({"audio": file["audio"], "channel": "all"})
return waveform
+4 -4
View File
@@ -81,10 +81,10 @@ def listen(audio_file: AudioFile, segment: Segment | None = None) -> None:
return
if segment is None:
waveform, sr = Audio(mono="downmix")(audio_file)
waveform, sr = Audio()(audio_file)
else:
waveform, sr = Audio(mono="downmix").crop(audio_file, segment)
return IPythonAudio(waveform.flatten(), rate=sr)
waveform, sr = Audio().crop(audio_file, segment)
return IPythonAudio(waveform.numpy(), rate=sr)
def preview(
@@ -157,7 +157,7 @@ def preview(
temp_dir = tempfile.mkdtemp(prefix="pyannote-audio-preview")
video_path = f"{temp_dir}/{uri}.{video_ext}"
audio = Audio(sample_rate=16000, mono="downmix")
audio = Audio(sample_rate=16000)
if segment is None:
duration = audio.get_duration(audio_file)
+1 -1
View File
@@ -28,7 +28,7 @@ from pyannote.database.protocol import SpeakerVerificationProtocol
from pyannote.audio.core.io import Audio
get_duration = Audio(mono="downmix").get_duration
get_duration = Audio().get_duration
def check_protocol(protocol: Protocol) -> Protocol:
+95 -10
View File
@@ -1,9 +1,12 @@
import random
import pytest
import torch
from pyannote.core import Segment
from torch import Tensor
from torchcodec.decoders import AudioDecoder, AudioStreamMetadata
from pyannote.audio.core.io import Audio
from pyannote.audio.core.io import Audio, Channel
def test_audio_resample():
@@ -11,7 +14,7 @@ def test_audio_resample():
test_file = "tests/data/dev00.wav"
metadata: AudioStreamMetadata = AudioDecoder(test_file).metadata
old_sr = metadata.sample_rate
loader = Audio(sample_rate=old_sr // 2, mono="downmix")
loader = Audio(sample_rate=old_sr // 2)
wav, sr = loader(test_file)
assert isinstance(wav, Tensor)
assert sr == old_sr // 2
@@ -19,7 +22,7 @@ def test_audio_resample():
def test_basic_load_with_defaults():
test_file = "tests/data/dev00.wav"
loader = Audio(mono="downmix")
loader = Audio()
wav, _sr = loader(test_file)
assert isinstance(wav, Tensor)
@@ -27,16 +30,98 @@ def test_basic_load_with_defaults():
def test_correct_audio_channel():
"When we specify an audio channel, it is chosen correctly"
waveform = torch.rand(2, 16000 * 2)
loader = Audio(mono="downmix")
loader = Audio()
wav, sr = loader({"waveform": waveform, "sample_rate": 16000, "channel": 1})
assert torch.equal(wav, waveform[1:2])
assert sr == 16000
@pytest.mark.parametrize("crop", [False, True])
@pytest.mark.parametrize("num_channels", [1, 3])
def test_random_audio_channel(monkeypatch, crop, num_channels):
waveform = torch.arange(num_channels).float().unsqueeze(1).repeat(1, 16000)
file = {"waveform": waveform, "sample_rate": 16000, "channel": "random"}
loader = Audio(sample_rate=8000)
rng = random.Random(42)
monkeypatch.setattr("pyannote.audio.core.io.random.randint", rng.randint)
expected_rng = random.Random(42)
for _ in range(5):
channel = expected_rng.randint(0, num_channels - 1)
selected = {**file, "channel": channel}
if crop:
actual, sr = loader.crop(file, Segment(0.25, 0.75))
expected, _ = loader.crop(selected, Segment(0.25, 0.75))
else:
actual, sr = loader(file)
expected, _ = loader(selected)
assert sr == 8000
assert actual.shape[0] == 1
torch.testing.assert_close(actual, expected)
@pytest.mark.parametrize("crop", [False, True])
@pytest.mark.parametrize("as_enum", [False, True])
@pytest.mark.parametrize("num_channels", [1, 3])
def test_all_audio_channels(crop, as_enum, num_channels):
waveform = torch.randn(num_channels, 16000)
file = {
"waveform": waveform,
"sample_rate": 16000,
"channel": Channel.ALL if as_enum else "all",
}
loader = Audio(sample_rate=8000)
if crop:
actual, sr = loader.crop(file, Segment(0.25, 0.75))
else:
actual, sr = loader(file)
# Each output channel must match independently loading and resampling that
# channel, preserving their original order.
expected = []
for channel in range(num_channels):
selected = {**file, "channel": channel}
if crop:
single, _ = loader.crop(selected, Segment(0.25, 0.75))
else:
single, _ = loader(selected)
expected.append(single)
assert sr == 8000
assert actual.shape[0] == num_channels
torch.testing.assert_close(actual, torch.cat(expected))
@pytest.mark.parametrize("crop", [False, True])
@pytest.mark.parametrize("as_enum", [False, True])
@pytest.mark.parametrize("num_channels", [1, 3])
def test_explicit_downmix(crop, as_enum, num_channels, monkeypatch):
waveform = torch.randn(num_channels, 16000)
file = {
"waveform": waveform,
"sample_rate": 16000,
"channel": Channel.DOWNMIX if as_enum else "downmix",
}
averaged = {"waveform": waveform.mean(dim=0, keepdim=True), "sample_rate": 16000}
loader = Audio(sample_rate=8000)
def unexpected_random(*args):
pytest.fail("Explicit downmix must not randomly select a channel")
monkeypatch.setattr("pyannote.audio.core.io.random.randint", unexpected_random)
if crop:
actual, sr = loader.crop(file, Segment(0.25, 0.75))
expected, _ = loader.crop(averaged, Segment(0.25, 0.75))
else:
actual, sr = loader(file)
expected, _ = loader(averaged)
assert sr == 8000
assert actual.shape[0] == 1
torch.testing.assert_close(actual, expected)
def test_can_load_with_waveform():
"We can load a raw waveform"
waveform = torch.rand(2, 16000 * 2)
loader = Audio(mono="downmix")
loader = Audio()
wav, sr = loader({"waveform": waveform, "sample_rate": 16000})
assert isinstance(wav, Tensor)
assert sr == 16000
@@ -45,7 +130,7 @@ def test_can_load_with_waveform():
def test_can_crop():
"Cropping works when we give a Segment"
test_file = "tests/data/dev00.wav"
loader = Audio(mono="downmix")
loader = Audio()
segment = Segment(0.2, 0.7)
wav, sr = loader.crop(test_file, segment)
assert wav.shape[1] / sr == 0.5
@@ -54,7 +139,7 @@ def test_can_crop():
def test_can_crop_waveform():
"Cropping works on raw waveforms"
waveform = torch.rand(1, 16000 * 2)
loader = Audio(mono="downmix")
loader = Audio()
segment = Segment(0.2, 0.7)
wav, sr = loader.crop({"waveform": waveform, "sample_rate": 16000}, segment)
assert isinstance(wav, Tensor)
@@ -64,7 +149,7 @@ def test_can_crop_waveform():
def test_can_crop_waveform_up_to_its_end():
"Cropping a raw waveform up to its very last sample does not raise"
waveform = torch.rand(1, 16000 * 2)
loader = Audio(mono="downmix")
loader = Audio()
segment = Segment(1.5, 2.0)
wav, _sr = loader.crop({"waveform": waveform, "sample_rate": 16000}, segment)
assert torch.equal(wav, waveform[:, 24000:])
@@ -73,7 +158,7 @@ def test_can_crop_waveform_up_to_its_end():
# File Like Object Tests
def test_can_load_from_file_like():
"Load entire wav of file like"
loader = Audio(mono="downmix")
loader = Audio()
with open("tests/data/dev00.wav", "rb") as f:
wav, sr = loader(f)
@@ -84,7 +169,7 @@ def test_can_load_from_file_like():
def test_can_crop_from_file_like():
"Load cropped sections from file like objects"
loader = Audio(mono="downmix")
loader = Audio()
with open("tests/data/dev00.wav", "rb") as f:
segment = Segment(0.2, 0.7)
+169
View File
@@ -0,0 +1,169 @@
import random
from types import SimpleNamespace
import pytest
import torch
from pyannote.core import Annotation, Segment, Timeline
from pyannote.database.protocol import SpeakerDiarizationProtocol
from pyannote.audio.core import io
from pyannote.audio.core.io import Channel
from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel
from pyannote.audio.tasks import SpeakerDiarization
class ChannelProtocol(SpeakerDiarizationProtocol):
name = "Test.SpeakerDiarization.Channels"
def __init__(self, audio_path, channels, validation_channels=None):
super().__init__()
self.audio_path = audio_path
self.channels = channels
self.validation_channels = (
channels if validation_channels is None else validation_channels
)
def files(self, subset):
channels = self.channels if subset == "train" else self.validation_channels
for index, channel in enumerate(channels):
annotation = Annotation()
annotation[Segment(0, 1)] = "speaker"
file = {
"uri": f"{subset}-{index}",
"database": f"Database{index}",
"subset": subset,
"scope": "file",
"audio": self.audio_path,
"annotation": annotation,
"annotated": Timeline([Segment(0, 1)]),
}
if channel is not None:
file["channel"] = channel
yield file
def train_iter(self):
yield from self.files("train")
def development_iter(self):
yield from self.files("development")
@pytest.mark.parametrize(
"channels",
[
(0,),
(2,),
(None,),
(None, 2, 0),
("random",),
("all",),
("downmix",),
(Channel.RANDOM, Channel.ALL, Channel.DOWNMIX),
(0, "random", None, 2, "all", "downmix"),
],
)
@pytest.mark.parametrize("stage", ["train", "development"])
def test_cached_channel_selection(tmp_path, monkeypatch, channels, stage):
# Mock only decoding: exercise the real crop, channel selection, and downmix.
# Distinct constant channels make selected and averaged audio distinguishable.
waveform = torch.tensor([0.125, 0.25, 0.75]).unsqueeze(1).repeat(1, 16000)
audio_path = tmp_path / "multichannel.wav"
audio_path.touch()
class Decoder:
def __init__(self, path):
assert str(path) == str(audio_path)
self.metadata = SimpleNamespace(
sample_rate=16000, duration_seconds_from_header=1.0
)
def get_samples_played_in_range(self, start, end):
return SimpleNamespace(
data=waveform[:, round(start * 16000) : round(end * 16000)],
sample_rate=16000,
)
monkeypatch.setattr(io, "AudioDecoder", Decoder, raising=False)
monkeypatch.setattr(io, "TORCHCODEC_AVAILABLE", True)
task = SpeakerDiarization(
ChannelProtocol(audio_path, channels),
cache=tmp_path / "task.npz",
duration=1.0,
max_speakers_per_chunk=2,
num_workers=0,
)
task.prepare_data()
task.setup()
assert "channel" in task.prepared_data["audio-metadata"].dtype.names
assert "channel" not in task.prepared_data["metadata-values"]
task.model = SimpleSegmentationModel()
for index, channel in enumerate(channels):
file_id = index + (len(channels) if stage == "development" else 0)
file = task.get_file(file_id)
if channel is None:
assert "channel" not in file
else:
assert file["channel"] == channel
selections = []
def select_channel(low, high):
assert (low, high) == (0, 2)
selected = (len(selections) + 1) % 3
selections.append(selected)
return selected
monkeypatch.setattr(io.random, "randint", select_channel)
if stage == "train":
# Select each source database in a compound protocol deterministically.
samples = task.train__iter__helper(
random.Random(0), database=f"Database{index}"
)
sample = next(samples)
else:
sample = task.val__getitem__(index)
if channel == "random":
torch.testing.assert_close(sample["X"], waveform[1:2])
second = next(samples) if stage == "train" else task.val__getitem__(index)
torch.testing.assert_close(second["X"], waveform[2:3])
assert selections == [1, 2]
continue
assert not selections
if channel == "all":
expected = waveform
elif channel is None or channel == "downmix":
expected = waveform.mean(dim=0, keepdim=True)
else:
expected = waveform[channel : channel + 1]
torch.testing.assert_close(sample["X"], expected)
torch.testing.assert_close(
task.collate_X([sample, sample]), torch.stack([expected, expected])
)
if stage == "development":
torch.testing.assert_close(task.val__getitem__(index)["X"], expected)
assert not selections
def make_task(tmp_path, validation_channels=(0,)):
return SpeakerDiarization(
ChannelProtocol(tmp_path / "audio.wav", ("random",), validation_channels),
cache=tmp_path / "task.npz",
duration=1.0,
max_speakers_per_chunk=2,
num_workers=0,
)
def test_channel_balancing_rejected(tmp_path):
task = make_task(tmp_path)
task.prepare_data()
task.setup()
task.model = SimpleSegmentationModel()
task.balance = ["channel"]
with pytest.raises(
ValueError, match="Channel selection cannot be used for balancing"
):
next(task.train__iter__())
+142
View File
@@ -0,0 +1,142 @@
"""Channel policies behave identically with raw, preloaded, and cached audio."""
import random
import wave
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from pyannote.core import Annotation, Segment
from pyannote.database.protocol import SpeakerVerificationProtocol
from pyannote.database.protocol.protocol import ProtocolFile
from pyannote.audio.core.io import Audio, Channel
from pyannote.audio.core.pipeline import Pipeline
from pyannote.audio.tasks.embedding.mixins import (
SupervisedRepresentationLearningTaskMixin,
)
from pyannote.audio.utils.preprocessors import Waveform
@pytest.fixture
def multichannel_file(tmp_path):
samples = np.tile(np.array([4096, 8192, 24576], dtype="<i2"), (16000, 1))
path = tmp_path / "channels.wav"
with wave.open(str(path), "wb") as output:
output.setnchannels(3)
output.setsampwidth(2)
output.setframerate(16000)
output.writeframes(samples.tobytes())
return {"audio": path}
@pytest.mark.parametrize("channel", [None, 0, 2, *Channel, "all", "downmix", "random"])
def test_preloading_preserves_policy(multichannel_file, channel, monkeypatch):
file = {**multichannel_file, "channel": channel}
pipeline = Pipeline()
raw = pipeline.prepare_one(dict(file), preload=False)
preloaded = pipeline.prepare_one(dict(file), preload=True)
assert preloaded["waveform"].shape == (3, 16000)
assert preloaded["channel"] == channel
loader = Audio(sample_rate=8000)
for cropped in (False, True):
# Identical random state must give identical selections with preloading.
outputs = []
for source in (raw, preloaded):
monkeypatch.setattr(
"pyannote.audio.core.io.random.randint", random.Random(7).randint
)
result = (
loader.crop(source, Segment(0.25, 0.75)) if cropped else loader(source)
)
outputs.append(result[0])
torch.testing.assert_close(*outputs)
@pytest.mark.parametrize("channel", [2, Channel.ALL, Channel.DOWNMIX])
def test_waveform_preprocessor_preserves_raw_channels(multichannel_file, channel):
file = ProtocolFile(
{**multichannel_file, "channel": channel, "sample_rate": 16000},
lazy={"waveform": Waveform()},
)
assert file["waveform"].shape == (3, 16000)
actual, _ = Audio()(file)
expected, _ = Audio()({**multichannel_file, "channel": channel})
torch.testing.assert_close(actual, expected)
@pytest.mark.parametrize("crop", [False, True])
@pytest.mark.parametrize("explicit_none", [False, True])
def test_missing_channel_defaults_to_downmix(crop, explicit_none):
waveform = torch.rand(3, 16000)
file = {"waveform": waveform, "sample_rate": 16000}
if explicit_none:
file["channel"] = None
loader = Audio()
actual, _ = loader.crop(file, Segment(0.25, 0.75)) if crop else loader(file)
expected = waveform.mean(dim=0, keepdim=True)
if crop:
expected = expected[:, 4000:12000]
torch.testing.assert_close(actual, expected)
@pytest.mark.parametrize("channel", [-1, -2, True, 1.5, "invalid", 3])
def test_invalid_channel_rejected(channel):
with pytest.raises(ValueError, match="channel"):
Audio()(
{"waveform": torch.ones(3, 16), "sample_rate": 16000, "channel": channel}
)
@pytest.mark.parametrize("channel", [0, Channel.RANDOM, Channel.ALL, Channel.DOWNMIX])
def test_embedding_training_keeps_channel(multichannel_file, channel):
annotation = Annotation()
annotation[Segment(0, 1)] = "speaker"
file = {
**multichannel_file,
"uri": "test",
"channel": channel,
"annotation": annotation,
}
task = SimpleNamespace(
protocol=SimpleNamespace(name="test", train=lambda: iter([file])),
min_duration=0.5,
duration=1,
)
SupervisedRepresentationLearningTaskMixin.setup(task)
assert task._train["speaker"][0]["channel"] == channel
@pytest.mark.parametrize("channel", ["random", Channel.RANDOM])
@pytest.mark.parametrize("duration", [0.5, 1.0])
def test_embedding_validation_supports_random(
multichannel_file, channel, duration, monkeypatch
):
file = {**multichannel_file, "channel": channel}
task = SimpleNamespace(
protocol=SpeakerVerificationProtocol(),
prepared_data={
"validation": [{"file1": file, "file2": file, "reference": True}]
},
model=SimpleNamespace(audio=Audio(sample_rate=16000)),
duration=duration,
)
selections = iter([1, 2, 2, 0])
calls = []
def select_channel(low, high):
assert (low, high) == (0, 2)
selected = next(selections)
calls.append(selected)
return selected
monkeypatch.setattr("pyannote.audio.core.io.random.randint", select_channel)
for expected_values in [(0.25, 0.75), (0.75, 0.125)]:
sample = SupervisedRepresentationLearningTaskMixin.val__getitem__(task, 0)
for index, value in enumerate(expected_values, start=1):
torch.testing.assert_close(
sample[f"X{index}"], torch.full((1, int(duration * 16000)), value)
)
assert sample["y"] is True
assert calls == [1, 2, 2, 0]
+93
View File
@@ -0,0 +1,93 @@
import sys
import wave
from types import SimpleNamespace
from unittest.mock import Mock
import numpy as np
import pytest
import torch
from pyannote.audio.core.io import Audio, Channel
from pyannote.audio.pipelines.pyannoteai.local import Local
from pyannote.audio.pipelines.pyannoteai.sdk import SDK
@pytest.fixture(params=["local", "sdk"])
def wrapper(request, monkeypatch):
backend = Mock()
if request.param == "local":
backend.diarize.return_value = {"diarization": [], "exclusive_diarization": []}
monkeypatch.setitem(
sys.modules,
"pyannoteai.local",
SimpleNamespace(Pipeline=lambda token: backend),
)
pipeline = Local(token="test")
payload = backend.diarize
else:
backend.upload.return_value = "media://test"
backend.diarize.return_value = "test-job"
backend.retrieve.return_value = {
"output": {"diarization": [], "exclusiveDiarization": []}
}
monkeypatch.setattr(
"pyannote.audio.pipelines.pyannoteai.sdk.Client", lambda token: backend
)
pipeline = SDK(token="test")
payload = backend.upload
return pipeline, backend, payload
@pytest.fixture
def audio_source(tmp_path):
waveform = torch.tensor([0.125, 0.25, 0.75]).unsqueeze(1).repeat(1, 16000)
path = tmp_path / "channels.wav"
with wave.open(str(path), "wb") as output:
output.setnchannels(3)
output.setsampwidth(2)
output.setframerate(16000)
output.writeframes((waveform.T.numpy() * 32768).astype("<i2").tobytes())
return path, waveform
@pytest.mark.parametrize("source", ["path", "waveform", "preloaded"])
@pytest.mark.parametrize("channel", [None, 0, 2, Channel.DOWNMIX, Channel.RANDOM])
def test_wrappers_apply_selection(wrapper, audio_source, source, channel, monkeypatch):
pipeline, _, payload = wrapper
path, waveform = audio_source
file = {"audio": path, "channel": channel}
if source == "waveform":
# Audio must prefer this waveform, rather than reread the path.
file.update(audio="missing.wav", waveform=waveform, sample_rate=16000)
monkeypatch.setattr("pyannote.audio.core.io.random.randint", lambda low, high: 2)
pipeline(file, preload=source == "preloaded", num_speakers=2)
processed = payload.call_args.args[0]
expected, _ = Audio()(
{"waveform": waveform, "sample_rate": 16000, "channel": channel}
)
assert set(processed) == {"waveform", "sample_rate"}
np.testing.assert_allclose(processed["waveform"], expected.numpy())
assert processed["sample_rate"] == 16000
assert processed["waveform"].shape == (1, 16000)
@pytest.mark.parametrize("source", ["path", "waveform", "preloaded"])
def test_wrappers_reject_multichannel_all(wrapper, audio_source, source):
pipeline, backend, _ = wrapper
path, waveform = audio_source
file = {"audio": path, "channel": Channel.ALL}
if source == "waveform":
file.update(waveform=waveform, sample_rate=16000)
with pytest.raises(ValueError, match="require mono audio"):
pipeline(file, preload=source == "preloaded")
backend.diarize.assert_not_called()
backend.upload.assert_not_called()
def test_wrappers_accept_all_for_mono(wrapper):
pipeline, _, payload = wrapper
waveform = torch.ones(1, 16000)
pipeline({"waveform": waveform, "sample_rate": 16000, "channel": Channel.ALL})
np.testing.assert_array_equal(
payload.call_args.args[0]["waveform"], waveform.numpy()
)
+2 -2
View File
@@ -211,7 +211,7 @@
"source": [
"from pyannote.audio import Audio\n",
"from IPython.display import Audio as IPythonAudio\n",
"waveform, sr = Audio(mono=\"downmix\").crop(DEMO_FILE, EXCERPT)\n",
"waveform, sr = Audio().crop(DEMO_FILE, EXCERPT)\n",
"IPythonAudio(waveform.flatten(), rate=sr)"
]
},
@@ -251,7 +251,7 @@
"notebook.reset()\n",
"\n",
"# load audio waveform and play it\n",
"waveform, sample_rate = Audio(mono=\"downmix\")(OWN_FILE)\n",
"waveform, sample_rate = Audio()(OWN_FILE)\n",
"IPythonAudio(data=waveform.squeeze(), rate=sample_rate, autoplay=True)"
]
},
+1 -1
View File
@@ -16,7 +16,7 @@
"\n",
"from pyannote.audio import Audio\n",
"from pyannote.core import Segment\n",
"audio = Audio(sample_rate=16000, mono=\"downmix\")\n",
"audio = Audio(sample_rate=16000)\n",
"\n",
"# extract embedding for a speaker speaking between t=3s and t=6s\n",
"speaker1 = Segment(3., 6.)\n",