diff --git a/CHANGELOG.md b/CHANGELOG.md index 26900371..011beb09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,11 @@ ## next +- BREAKING(io): rename `Audio.downmix_and_resample` to `Audio.select_channels_and_resample` +- BREAKING(io): remove `Audio(mono=...)`; use the file's `channel` policy instead. - BREAKING(cli): remove gap-filling optimization (`--optimize`) and `OptimizedMinDurationOff` outputs from the `benchmark` command - BREAKING: require Python 3.11 or newer; Python 3.10 is no longer supported +- feat(task): support `channel: {int | downmix | random | all}` for audio loading (requires `pyannote.database` >= 6.3.0) - fix(model): restore `fbank_only` mode in WeSpeaker embedding models - improve(model): compute WeSpeaker fbank features on MPS instead of falling back to CPU [@Isalia20](https://github.com/Isalia20) - fix(task): avoid training crashes with `spawn`/`forkserver` workers, including diff --git a/notebook/augmentation.ipynb b/notebook/augmentation.ipynb index 656eae4a..93ef4dbe 100644 --- a/notebook/augmentation.ipynb +++ b/notebook/augmentation.ipynb @@ -12,7 +12,7 @@ " preprocessors={\"audio\": FileFinder()})\n", "\n", "from pyannote.audio.core.io import Audio\n", - "audio = Audio(sample_rate=16000, mono=\"downmix\")\n", + "audio = Audio(sample_rate=16000)\n", "file = next(protocol.test())\n", "\n", "from pyannote.core import Segment\n", diff --git a/pyproject.toml b/pyproject.toml index 57c8da43..27e149b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ "opentelemetry-sdk>=1.34.0", "opentelemetry-exporter-otlp>=1.34.0", "pyannote-core>=6.1.0", - "pyannote-database>=6.1.1", + "pyannote-database>=6.3.0", "pyannote-metrics>=4.2.0", "pyannote-pipeline>=4.1.0", "pytorch-metric-learning>=2.8.1", diff --git a/src/pyannote/audio/core/io.py b/src/pyannote/audio/core/io.py index faab56db..732f98ce 100644 --- a/src/pyannote/audio/core/io.py +++ b/src/pyannote/audio/core/io.py @@ -31,7 +31,9 @@ pyannote.audio relies on torchcodec for reading and torchaudio for resampling. import random import warnings from collections.abc import Mapping +from enum import StrEnum from io import IOBase +from numbers import Integral from pathlib import Path import torch.nn.functional as F @@ -56,6 +58,50 @@ except Exception as e: AudioStreamMetadata = None +class Channel(StrEnum): + """Channel selection options""" + + RANDOM = "random" + ALL = "all" + DOWNMIX = "downmix" + + @classmethod + def normalize(cls, channel) -> "int | Channel": + """Normalize channel selection + + Parameters + ---------- + channel : int, str, Channel, or None + Zero-based channel index or channel selection option. + Supported options are "random", "all", and "downmix". + None defaults to "downmix". + + Returns + ------- + channel : int or Channel + Non-negative Python integer or Channel enum member. + + Raises + ------ + ValueError + If channel is not a non-negative integer, a supported option, or None. + + """ + if channel is None: + return cls.DOWNMIX + if isinstance(channel, Integral) and not isinstance(channel, bool): + if channel >= 0: + return int(channel) + elif isinstance(channel, str): + try: + return cls(channel) + except ValueError: + pass + raise ValueError( + 'channel must be a non-negative integer, "random", "all", or "downmix".' + ) + + AudioFile = str | Path | IOBase | Mapping AudioFileDocString = """ @@ -68,6 +114,9 @@ Audio files can be provided to the Audio class using different types: For last two options, an additional "channel" key can be provided as a zero-indexed integer to load a specific channel: {"audio": "stereo.wav", "channel": 0} +or "random" to select a channel randomly on each load or crop. Use "all" to +preserve all channels in their original order. Use "downmix" (the default when +channel is omitted or None) to average all channels. """ @@ -114,14 +163,14 @@ class Audio: ---------- sample_rate: int, optional Target sampling rate. Defaults to using native sampling rate. - mono : {'random', 'downmix'}, optional - In case of multi-channel audio, convert to single-channel audio - using one of the following strategies: select one channel at - 'random' or 'downmix' by averaging all channels. + + Notes + ----- + Channel selection is controlled by the file dictionary, defaulting to "downmix". Usage ----- - >>> audio = Audio(sample_rate=16000, mono='downmix') + >>> audio = Audio(sample_rate=16000) >>> waveform, sample_rate = audio({"audio": "/path/to/audio.wav"}) >>> assert sample_rate == 16000 >>> sample_rate = 44100 @@ -215,15 +264,17 @@ class Audio: return file - def __init__(self, sample_rate: int | None = None, mono=None): + def __init__(self, sample_rate: int | None = None): super().__init__() self.sample_rate = sample_rate - self.mono = mono - def downmix_and_resample( - self, waveform: Tensor, sample_rate: int, channel: int | None = None + def select_channels_and_resample( + self, + waveform: Tensor, + sample_rate: int, + channel: int | str | Channel | None = None, ) -> tuple[Tensor, int]: - """Downmix and resample + """Apply the channel policy and resample Parameters ---------- @@ -231,29 +282,33 @@ class Audio: Waveform. sample_rate : int Sample rate. - channel : int, optional - Channel to use. + channel : int, "random", "all", or "downmix", optional + Zero-based channel to use, "random" to select one on each call, or + "all" to preserve every channel. "downmix" averages all channels. + Defaults to "downmix" when omitted or None. Returns ------- waveform : (channel, time) Tensor - Remixed and resampled waveform + Waveform after applying the channel policy and resampling sample_rate : int New sample rate """ - # select channel if requested - if channel is not None: - waveform = waveform[channel : channel + 1] - - # downmix to mono + channel = Channel.normalize(channel) num_channels = waveform.shape[0] - if num_channels > 1: - if self.mono == "random": - channel = random.randint(0, num_channels - 1) - waveform = waveform[channel : channel + 1] - elif self.mono == "downmix": - waveform = waveform.mean(dim=0, keepdim=True) + if num_channels < 1: + raise ValueError("Audio must contain at least one channel.") + if channel == Channel.RANDOM: + channel = random.randint(0, num_channels - 1) + if isinstance(channel, int): + if channel >= num_channels: + raise ValueError( + f"Requested channel {channel}, but audio has {num_channels} channels." + ) + waveform = waveform[channel : channel + 1] + elif channel == Channel.DOWNMIX and num_channels > 1: + waveform = waveform.mean(dim=0, keepdim=True) # resample if (self.sample_rate is not None) and (self.sample_rate != sample_rate): @@ -327,7 +382,9 @@ class Audio: waveform = file["waveform"] sample_rate = file["sample_rate"] - return self.downmix_and_resample(waveform, sample_rate, channel=channel) + return self.select_channels_and_resample( + waveform, sample_rate, channel=channel + ) if not TORCHCODEC_AVAILABLE: raise RuntimeError( @@ -346,7 +403,7 @@ class Audio: if isinstance(file["audio"], IOBase): file["audio"].seek(0) - return self.downmix_and_resample(waveform, sample_rate, channel=channel) + return self.select_channels_and_resample(waveform, sample_rate, channel=channel) def crop( self, @@ -409,7 +466,7 @@ class Audio: data = waveform[:, start_sample:end_sample] data = F.pad(data, (pad_start, pad_end)) - return self.downmix_and_resample(data, sample_rate, channel=channel) + return self.select_channels_and_resample(data, sample_rate, channel=channel) if not TORCHCODEC_AVAILABLE: raise RuntimeError( @@ -479,4 +536,4 @@ class Audio: pad_end += 1 data = F.pad(data, (pad_start, pad_end)) - return self.downmix_and_resample(data, sample_rate, channel=channel) + return self.select_channels_and_resample(data, sample_rate, channel=channel) diff --git a/src/pyannote/audio/core/model.py b/src/pyannote/audio/core/model.py index 3fa9c914..5040e586 100644 --- a/src/pyannote/audio/core/model.py +++ b/src/pyannote/audio/core/model.py @@ -88,14 +88,13 @@ class Model(lightning.LightningModule): ): super().__init__() - assert num_channels == 1, ( - "Only mono audio is supported for now (num_channels = 1)" - ) + if not isinstance(num_channels, int) or num_channels < 1: + raise ValueError("num_channels must be a positive integer.") self.save_hyperparameters("sample_rate", "num_channels") self.task = task - self.audio = Audio(sample_rate=self.hparams.sample_rate, mono="downmix") + self.audio = Audio(sample_rate=self.hparams.sample_rate) @property def task(self) -> Task: diff --git a/src/pyannote/audio/core/pipeline.py b/src/pyannote/audio/core/pipeline.py index 463bbbb5..b7adc259 100644 --- a/src/pyannote/audio/core/pipeline.py +++ b/src/pyannote/audio/core/pipeline.py @@ -482,12 +482,9 @@ class Pipeline(_Pipeline): "Cannot preload audio: `waveform` key is already available or will be via a preprocessor." ) - # load waveform in memory (and keep track of its original sample rate) - file["waveform"], file["sample_rate"] = Audio()(file) - - # the above line already took care of channel selection, - # therefore we remove the `channel` key from the file - file.pop("channel", None) + # load waveform in memory and keep track of its original sample rate. + # keep all channels for now: channel selection is done later at load/crop time. + file["waveform"], file["sample_rate"] = Audio()({**file, "channel": "all"}) return file diff --git a/src/pyannote/audio/core/task.py b/src/pyannote/audio/core/task.py index fc42d79f..f4922d79 100644 --- a/src/pyannote/audio/core/task.py +++ b/src/pyannote/audio/core/task.py @@ -32,7 +32,7 @@ import warnings from collections import defaultdict from collections.abc import Sequence from dataclasses import dataclass -from enum import Enum +from enum import Enum, IntEnum from functools import cached_property, partial from numbers import Number from pathlib import Path @@ -50,12 +50,24 @@ from torch_audiomentations import Identity from torch_audiomentations.core.transforms_interface import BaseWaveformTransform from torchmetrics import Metric, MetricCollection +from pyannote.audio.core.io import Channel from pyannote.audio.utils.loss import binary_cross_entropy, nll_loss from pyannote.audio.utils.protocol import check_protocol Subsets = list(Subset.__args__) Scopes = list(Scope.__args__) +MISSING_METADATA = -1 + + +class _CachedChannel(IntEnum): + """Private integer encoding for channel selection in task caches.""" + + UNSPECIFIED = MISSING_METADATA + RANDOM = -2 + ALL = -3 + DOWNMIX = -4 + # Type of machine learning problem class Problem(Enum): @@ -332,7 +344,7 @@ class Task(lightning.LightningDataModule): prepared_data = { 'protocol': name of the protocol 'audio-path': array of N paths to audio - 'audio-metadata': array of N audio infos such as audio subset, scope and database + 'audio-metadata': array of N audio infos such as audio channel, subset, scope and database. 'audio-annotated': array of N annotated duration (usually equals file duration but might be shorter if file is not fully annotated) 'annotations-regions': array of M annotated regions 'audio-regions-ids': array of N start/end indices of annotated regions @@ -402,6 +414,16 @@ class Task(lightning.LightningDataModule): # keep track of label scope (file, database, or global) metadatum["scope"] = Scopes.index(file["scope"]) + # Channel selection is a loading option, not categorical metadata. + metadatum["channel"] = _CachedChannel.UNSPECIFIED + if "channel" in file: + channel = Channel.normalize(file["channel"]) + metadatum["channel"] = ( + _CachedChannel[channel.name] + if isinstance(channel, Channel) + else channel + ) + remaining_metadata_keys = set(file) - { "uri", "database", @@ -409,6 +431,7 @@ class Task(lightning.LightningDataModule): "audio", "scope", "classes", + "channel", "annotation", "annotated", } @@ -525,14 +548,16 @@ class Task(lightning.LightningDataModule): # append segment IDs (start and end time) audio_segments_ids.append((_segments_id, segments_id)) - # since not all metadata keys are present in all files, fallback to -1 when a key is missing + # Register channel separately without collecting its possible values. + metadata_keys = [*metadata_unique_values, "channel"] + # Not all metadata keys are present in every file. metadata = [ - tuple(metadatum.get(key, -1) for key in metadata_unique_values) + tuple(metadatum.get(key, MISSING_METADATA) for key in metadata_keys) for metadatum in metadata ] metadata_dtype = [ (key, get_dtype(max(m[i] for m in metadata))) - for i, key in enumerate(metadata_unique_values) + for i, key in enumerate(metadata_keys) ] # turn list of annotated regions into a single numpy array diff --git a/src/pyannote/audio/pipelines/pyannoteai/__init__.py b/src/pyannote/audio/pipelines/pyannoteai/__init__.py index d598a73a..6e31fb2a 100644 --- a/src/pyannote/audio/pipelines/pyannoteai/__init__.py +++ b/src/pyannote/audio/pipelines/pyannoteai/__init__.py @@ -20,3 +20,32 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. + +from pyannote.audio.core.io import Audio, AudioFile + + +def _prepare_audio(file: AudioFile) -> dict: + """Apply channel selection before handing audio to a mono-only backend. + + Parameters + ---------- + file : AudioFile + Audio file with an optional channel selection policy. + + Returns + ------- + audio : dict + Selected mono waveform as a NumPy array and its sample rate. + + Raises + ------ + ValueError + If the selected waveform contains more than one channel. + """ + waveform, sample_rate = Audio()(file) + if waveform.shape[0] != 1: + raise ValueError( + 'pyannoteAI backends require mono audio. Use channel="downmix", ' + '"random", or a fixed channel index instead of "all" for multi-channel audio.' + ) + return {"waveform": waveform.numpy(force=True), "sample_rate": sample_rate} diff --git a/src/pyannote/audio/pipelines/pyannoteai/local.py b/src/pyannote/audio/pipelines/pyannoteai/local.py index 0afbae01..ff2a7b61 100644 --- a/src/pyannote/audio/pipelines/pyannoteai/local.py +++ b/src/pyannote/audio/pipelines/pyannoteai/local.py @@ -28,6 +28,7 @@ from pyannote.audio import Pipeline from pyannote.audio.core.io import AudioFile from ..speaker_diarization import DiarizeOutput +from . import _prepare_audio class Local(Pipeline): @@ -89,33 +90,20 @@ class Local(Pipeline): ------- output : DiarizeOutput DiarizeOutput object containing both regular and exclusive speaker diarization results. + + Raises + ------ + ValueError + If channel selection leaves multiple channels. The backend requires mono audio. """ - # if file provides "audio" path - if "audio" in file: - predictions = self._pipeline.diarize( - file["audio"], - num_speakers=num_speakers, - min_speakers=min_speakers, - max_speakers=max_speakers, - **kwargs, - ) - - # if file provides "waveform", make sure it is numpy (and not torch) array - elif "waveform" in file: - waveform = file["waveform"] - if hasattr(waveform, "numpy"): - waveform = waveform.numpy(force=True) - - predictions = self._pipeline.diarize( - {"waveform": waveform, "sample_rate": file["sample_rate"]}, - num_speakers=num_speakers, - min_speakers=min_speakers, - max_speakers=max_speakers, - **kwargs, - ) - else: - raise ValueError("AudioFile must provide either 'audio' or 'waveform' key") + predictions = self._pipeline.diarize( + _prepare_audio(file), + num_speakers=num_speakers, + min_speakers=min_speakers, + max_speakers=max_speakers, + **kwargs, + ) speaker_diarization: Annotation = self._deserialize(predictions["diarization"]) exclusive_speaker_diarization: Annotation = self._deserialize( diff --git a/src/pyannote/audio/pipelines/pyannoteai/sdk.py b/src/pyannote/audio/pipelines/pyannoteai/sdk.py index 022c5a2b..dc62bb2d 100644 --- a/src/pyannote/audio/pipelines/pyannoteai/sdk.py +++ b/src/pyannote/audio/pipelines/pyannoteai/sdk.py @@ -29,6 +29,8 @@ from pyannote.audio import Pipeline from pyannote.audio.core.io import AudioFile from pyannote.audio.pipelines.speaker_diarization import DiarizeOutput +from . import _prepare_audio + class SDK(Pipeline): """Wrapper around official pyannoteAI API client @@ -97,6 +99,8 @@ class SDK(Pipeline): Raises ------ + ValueError + If channel selection leaves multiple channels. The backend requires mono audio. PyannoteAIFailedJob If the job failed PyannoteAICanceledJob @@ -106,7 +110,7 @@ class SDK(Pipeline): """ # upload file to pyannoteAI cloud API - media_url: str = self._client.upload(file) + media_url: str = self._client.upload(_prepare_audio(file)) # initiate diarization job job_id = self._client.diarize( diff --git a/src/pyannote/audio/pipelines/speaker_diarization.py b/src/pyannote/audio/pipelines/speaker_diarization.py index cfe16ff6..333c745c 100644 --- a/src/pyannote/audio/pipelines/speaker_diarization.py +++ b/src/pyannote/audio/pipelines/speaker_diarization.py @@ -268,7 +268,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): self._embedding = PretrainedSpeakerEmbedding( self.embedding, token=token, cache_dir=cache_dir ) - self._audio = Audio(sample_rate=self._embedding.sample_rate, mono="downmix") + self._audio = Audio(sample_rate=self._embedding.sample_rate) metric = self._embedding.metric try: diff --git a/src/pyannote/audio/pipelines/speech_separation.py b/src/pyannote/audio/pipelines/speech_separation.py index 3cac7008..7a78f6dd 100644 --- a/src/pyannote/audio/pipelines/speech_separation.py +++ b/src/pyannote/audio/pipelines/speech_separation.py @@ -180,7 +180,7 @@ class SpeechSeparation(SpeakerDiarizationMixin, Pipeline): self._embedding = PretrainedSpeakerEmbedding( self.embedding, token=token, cache_dir=cache_dir ) - self._audio = Audio(sample_rate=self._embedding.sample_rate, mono="downmix") + self._audio = Audio(sample_rate=self._embedding.sample_rate) metric = self._embedding.metric try: diff --git a/src/pyannote/audio/pipelines/utils/oracle.py b/src/pyannote/audio/pipelines/utils/oracle.py index bff6965e..e630c989 100644 --- a/src/pyannote/audio/pipelines/utils/oracle.py +++ b/src/pyannote/audio/pipelines/utils/oracle.py @@ -61,7 +61,7 @@ def oracle_segmentation( """ if "duration" not in file: - duration = Audio(mono="downmix").get_duration(file) + duration = Audio().get_duration(file) else: duration: float = file["duration"] reference: Annotation = file["annotation"] diff --git a/src/pyannote/audio/tasks/embedding/mixins.py b/src/pyannote/audio/tasks/embedding/mixins.py index 5433698f..ce5e38e4 100644 --- a/src/pyannote/audio/tasks/embedding/mixins.py +++ b/src/pyannote/audio/tasks/embedding/mixins.py @@ -36,6 +36,7 @@ from torchmetrics import Metric from torchmetrics.classification import BinaryAUROC from tqdm import tqdm +from pyannote.audio.core.io import Channel from pyannote.audio.core.task import Problem, Resolution, Specifications from pyannote.audio.torchmetrics.classification import EqualErrorRate from pyannote.audio.utils.random import create_rng_for_worker @@ -107,6 +108,7 @@ class SupervisedRepresentationLearningTaskMixin: { "uri": f["uri"], "audio": f["audio"], + "channel": Channel.normalize(f.get("channel")), "duration": duration, "speech_turns": speech_turns, } diff --git a/src/pyannote/audio/tasks/segmentation/mixins.py b/src/pyannote/audio/tasks/segmentation/mixins.py index 86ed745c..c2321cb7 100644 --- a/src/pyannote/audio/tasks/segmentation/mixins.py +++ b/src/pyannote/audio/tasks/segmentation/mixins.py @@ -36,7 +36,8 @@ from torch.utils.data._utils.collate import default_collate from torchmetrics import Metric from torchmetrics.classification import BinaryAUROC, MulticlassAUROC, MultilabelAUROC -from pyannote.audio.core.task import Problem, Task, get_dtype +from pyannote.audio.core.io import Channel +from pyannote.audio.core.task import Problem, Task, _CachedChannel, get_dtype from pyannote.audio.utils.random import create_rng_for_worker Subsets = list(Subset.__args__) @@ -47,7 +48,20 @@ class SegmentationTask(Task): """Methods common to most segmentation tasks""" def get_file(self, file_id): - return {"audio": self.prepared_data["audio-path"][file_id]} + file = {"audio": self.prepared_data["audio-path"][file_id]} + metadata = self.prepared_data["audio-metadata"][file_id] + # Channels are cached as literal indices or named _CachedChannel modes. + if "channel" in metadata.dtype.names: + channel = int(metadata["channel"]) + if channel == _CachedChannel.RANDOM: + file["channel"] = Channel.RANDOM + elif channel == _CachedChannel.ALL: + file["channel"] = Channel.ALL + elif channel == _CachedChannel.DOWNMIX: + file["channel"] = Channel.DOWNMIX + elif channel >= 0: + file["channel"] = channel + return file def default_metric( self, @@ -157,6 +171,8 @@ class SegmentationTask(Task): rng = create_rng_for_worker(self.model) balance = getattr(self, "balance", None) + if balance is not None and "channel" in balance: + raise ValueError("Channel selection cannot be used for balancing.") if balance is None: chunks = self.train__iter__helper(rng) diff --git a/src/pyannote/audio/utils/preprocessors.py b/src/pyannote/audio/utils/preprocessors.py index 8637d0a0..0f14b8fc 100644 --- a/src/pyannote/audio/utils/preprocessors.py +++ b/src/pyannote/audio/utils/preprocessors.py @@ -132,7 +132,8 @@ class Waveform: self._audio = Audio() def __call__(self, file: ProtocolFile): - waveform, _ = self._audio(file) + # Preprocessors decode raw audio; consumers apply the channel policy. + waveform, _ = self._audio({"audio": file["audio"], "channel": "all"}) return waveform diff --git a/src/pyannote/audio/utils/preview.py b/src/pyannote/audio/utils/preview.py index 351b2258..049da064 100644 --- a/src/pyannote/audio/utils/preview.py +++ b/src/pyannote/audio/utils/preview.py @@ -81,10 +81,10 @@ def listen(audio_file: AudioFile, segment: Segment | None = None) -> None: return if segment is None: - waveform, sr = Audio(mono="downmix")(audio_file) + waveform, sr = Audio()(audio_file) else: - waveform, sr = Audio(mono="downmix").crop(audio_file, segment) - return IPythonAudio(waveform.flatten(), rate=sr) + waveform, sr = Audio().crop(audio_file, segment) + return IPythonAudio(waveform.numpy(), rate=sr) def preview( @@ -157,7 +157,7 @@ def preview( temp_dir = tempfile.mkdtemp(prefix="pyannote-audio-preview") video_path = f"{temp_dir}/{uri}.{video_ext}" - audio = Audio(sample_rate=16000, mono="downmix") + audio = Audio(sample_rate=16000) if segment is None: duration = audio.get_duration(audio_file) diff --git a/src/pyannote/audio/utils/protocol.py b/src/pyannote/audio/utils/protocol.py index 0d493ede..c6427af0 100644 --- a/src/pyannote/audio/utils/protocol.py +++ b/src/pyannote/audio/utils/protocol.py @@ -28,7 +28,7 @@ from pyannote.database.protocol import SpeakerVerificationProtocol from pyannote.audio.core.io import Audio -get_duration = Audio(mono="downmix").get_duration +get_duration = Audio().get_duration def check_protocol(protocol: Protocol) -> Protocol: diff --git a/tests/io_test.py b/tests/io_test.py index 4a6c9065..49d29c06 100644 --- a/tests/io_test.py +++ b/tests/io_test.py @@ -1,9 +1,12 @@ +import random + +import pytest import torch from pyannote.core import Segment from torch import Tensor from torchcodec.decoders import AudioDecoder, AudioStreamMetadata -from pyannote.audio.core.io import Audio +from pyannote.audio.core.io import Audio, Channel def test_audio_resample(): @@ -11,7 +14,7 @@ def test_audio_resample(): test_file = "tests/data/dev00.wav" metadata: AudioStreamMetadata = AudioDecoder(test_file).metadata old_sr = metadata.sample_rate - loader = Audio(sample_rate=old_sr // 2, mono="downmix") + loader = Audio(sample_rate=old_sr // 2) wav, sr = loader(test_file) assert isinstance(wav, Tensor) assert sr == old_sr // 2 @@ -19,7 +22,7 @@ def test_audio_resample(): def test_basic_load_with_defaults(): test_file = "tests/data/dev00.wav" - loader = Audio(mono="downmix") + loader = Audio() wav, _sr = loader(test_file) assert isinstance(wav, Tensor) @@ -27,16 +30,98 @@ def test_basic_load_with_defaults(): def test_correct_audio_channel(): "When we specify an audio channel, it is chosen correctly" waveform = torch.rand(2, 16000 * 2) - loader = Audio(mono="downmix") + loader = Audio() wav, sr = loader({"waveform": waveform, "sample_rate": 16000, "channel": 1}) assert torch.equal(wav, waveform[1:2]) assert sr == 16000 +@pytest.mark.parametrize("crop", [False, True]) +@pytest.mark.parametrize("num_channels", [1, 3]) +def test_random_audio_channel(monkeypatch, crop, num_channels): + waveform = torch.arange(num_channels).float().unsqueeze(1).repeat(1, 16000) + file = {"waveform": waveform, "sample_rate": 16000, "channel": "random"} + loader = Audio(sample_rate=8000) + rng = random.Random(42) + monkeypatch.setattr("pyannote.audio.core.io.random.randint", rng.randint) + expected_rng = random.Random(42) + for _ in range(5): + channel = expected_rng.randint(0, num_channels - 1) + selected = {**file, "channel": channel} + if crop: + actual, sr = loader.crop(file, Segment(0.25, 0.75)) + expected, _ = loader.crop(selected, Segment(0.25, 0.75)) + else: + actual, sr = loader(file) + expected, _ = loader(selected) + assert sr == 8000 + assert actual.shape[0] == 1 + torch.testing.assert_close(actual, expected) + + +@pytest.mark.parametrize("crop", [False, True]) +@pytest.mark.parametrize("as_enum", [False, True]) +@pytest.mark.parametrize("num_channels", [1, 3]) +def test_all_audio_channels(crop, as_enum, num_channels): + waveform = torch.randn(num_channels, 16000) + file = { + "waveform": waveform, + "sample_rate": 16000, + "channel": Channel.ALL if as_enum else "all", + } + loader = Audio(sample_rate=8000) + if crop: + actual, sr = loader.crop(file, Segment(0.25, 0.75)) + else: + actual, sr = loader(file) + + # Each output channel must match independently loading and resampling that + # channel, preserving their original order. + expected = [] + for channel in range(num_channels): + selected = {**file, "channel": channel} + if crop: + single, _ = loader.crop(selected, Segment(0.25, 0.75)) + else: + single, _ = loader(selected) + expected.append(single) + assert sr == 8000 + assert actual.shape[0] == num_channels + torch.testing.assert_close(actual, torch.cat(expected)) + + +@pytest.mark.parametrize("crop", [False, True]) +@pytest.mark.parametrize("as_enum", [False, True]) +@pytest.mark.parametrize("num_channels", [1, 3]) +def test_explicit_downmix(crop, as_enum, num_channels, monkeypatch): + waveform = torch.randn(num_channels, 16000) + file = { + "waveform": waveform, + "sample_rate": 16000, + "channel": Channel.DOWNMIX if as_enum else "downmix", + } + averaged = {"waveform": waveform.mean(dim=0, keepdim=True), "sample_rate": 16000} + loader = Audio(sample_rate=8000) + + def unexpected_random(*args): + pytest.fail("Explicit downmix must not randomly select a channel") + + monkeypatch.setattr("pyannote.audio.core.io.random.randint", unexpected_random) + if crop: + actual, sr = loader.crop(file, Segment(0.25, 0.75)) + expected, _ = loader.crop(averaged, Segment(0.25, 0.75)) + else: + actual, sr = loader(file) + expected, _ = loader(averaged) + assert sr == 8000 + assert actual.shape[0] == 1 + torch.testing.assert_close(actual, expected) + + def test_can_load_with_waveform(): "We can load a raw waveform" waveform = torch.rand(2, 16000 * 2) - loader = Audio(mono="downmix") + loader = Audio() wav, sr = loader({"waveform": waveform, "sample_rate": 16000}) assert isinstance(wav, Tensor) assert sr == 16000 @@ -45,7 +130,7 @@ def test_can_load_with_waveform(): def test_can_crop(): "Cropping works when we give a Segment" test_file = "tests/data/dev00.wav" - loader = Audio(mono="downmix") + loader = Audio() segment = Segment(0.2, 0.7) wav, sr = loader.crop(test_file, segment) assert wav.shape[1] / sr == 0.5 @@ -54,7 +139,7 @@ def test_can_crop(): def test_can_crop_waveform(): "Cropping works on raw waveforms" waveform = torch.rand(1, 16000 * 2) - loader = Audio(mono="downmix") + loader = Audio() segment = Segment(0.2, 0.7) wav, sr = loader.crop({"waveform": waveform, "sample_rate": 16000}, segment) assert isinstance(wav, Tensor) @@ -64,7 +149,7 @@ def test_can_crop_waveform(): def test_can_crop_waveform_up_to_its_end(): "Cropping a raw waveform up to its very last sample does not raise" waveform = torch.rand(1, 16000 * 2) - loader = Audio(mono="downmix") + loader = Audio() segment = Segment(1.5, 2.0) wav, _sr = loader.crop({"waveform": waveform, "sample_rate": 16000}, segment) assert torch.equal(wav, waveform[:, 24000:]) @@ -73,7 +158,7 @@ def test_can_crop_waveform_up_to_its_end(): # File Like Object Tests def test_can_load_from_file_like(): "Load entire wav of file like" - loader = Audio(mono="downmix") + loader = Audio() with open("tests/data/dev00.wav", "rb") as f: wav, sr = loader(f) @@ -84,7 +169,7 @@ def test_can_load_from_file_like(): def test_can_crop_from_file_like(): "Load cropped sections from file like objects" - loader = Audio(mono="downmix") + loader = Audio() with open("tests/data/dev00.wav", "rb") as f: segment = Segment(0.2, 0.7) diff --git a/tests/tasks/test_channel_selection.py b/tests/tasks/test_channel_selection.py new file mode 100644 index 00000000..b1aaf387 --- /dev/null +++ b/tests/tasks/test_channel_selection.py @@ -0,0 +1,169 @@ +import random +from types import SimpleNamespace + +import pytest +import torch +from pyannote.core import Annotation, Segment, Timeline +from pyannote.database.protocol import SpeakerDiarizationProtocol + +from pyannote.audio.core import io +from pyannote.audio.core.io import Channel +from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel +from pyannote.audio.tasks import SpeakerDiarization + + +class ChannelProtocol(SpeakerDiarizationProtocol): + name = "Test.SpeakerDiarization.Channels" + + def __init__(self, audio_path, channels, validation_channels=None): + super().__init__() + self.audio_path = audio_path + self.channels = channels + self.validation_channels = ( + channels if validation_channels is None else validation_channels + ) + + def files(self, subset): + channels = self.channels if subset == "train" else self.validation_channels + for index, channel in enumerate(channels): + annotation = Annotation() + annotation[Segment(0, 1)] = "speaker" + file = { + "uri": f"{subset}-{index}", + "database": f"Database{index}", + "subset": subset, + "scope": "file", + "audio": self.audio_path, + "annotation": annotation, + "annotated": Timeline([Segment(0, 1)]), + } + if channel is not None: + file["channel"] = channel + yield file + + def train_iter(self): + yield from self.files("train") + + def development_iter(self): + yield from self.files("development") + + +@pytest.mark.parametrize( + "channels", + [ + (0,), + (2,), + (None,), + (None, 2, 0), + ("random",), + ("all",), + ("downmix",), + (Channel.RANDOM, Channel.ALL, Channel.DOWNMIX), + (0, "random", None, 2, "all", "downmix"), + ], +) +@pytest.mark.parametrize("stage", ["train", "development"]) +def test_cached_channel_selection(tmp_path, monkeypatch, channels, stage): + # Mock only decoding: exercise the real crop, channel selection, and downmix. + # Distinct constant channels make selected and averaged audio distinguishable. + waveform = torch.tensor([0.125, 0.25, 0.75]).unsqueeze(1).repeat(1, 16000) + audio_path = tmp_path / "multichannel.wav" + audio_path.touch() + + class Decoder: + def __init__(self, path): + assert str(path) == str(audio_path) + self.metadata = SimpleNamespace( + sample_rate=16000, duration_seconds_from_header=1.0 + ) + + def get_samples_played_in_range(self, start, end): + return SimpleNamespace( + data=waveform[:, round(start * 16000) : round(end * 16000)], + sample_rate=16000, + ) + + monkeypatch.setattr(io, "AudioDecoder", Decoder, raising=False) + monkeypatch.setattr(io, "TORCHCODEC_AVAILABLE", True) + task = SpeakerDiarization( + ChannelProtocol(audio_path, channels), + cache=tmp_path / "task.npz", + duration=1.0, + max_speakers_per_chunk=2, + num_workers=0, + ) + task.prepare_data() + task.setup() + assert "channel" in task.prepared_data["audio-metadata"].dtype.names + assert "channel" not in task.prepared_data["metadata-values"] + task.model = SimpleSegmentationModel() + + for index, channel in enumerate(channels): + file_id = index + (len(channels) if stage == "development" else 0) + file = task.get_file(file_id) + if channel is None: + assert "channel" not in file + else: + assert file["channel"] == channel + + selections = [] + + def select_channel(low, high): + assert (low, high) == (0, 2) + selected = (len(selections) + 1) % 3 + selections.append(selected) + return selected + + monkeypatch.setattr(io.random, "randint", select_channel) + if stage == "train": + # Select each source database in a compound protocol deterministically. + samples = task.train__iter__helper( + random.Random(0), database=f"Database{index}" + ) + sample = next(samples) + else: + sample = task.val__getitem__(index) + + if channel == "random": + torch.testing.assert_close(sample["X"], waveform[1:2]) + second = next(samples) if stage == "train" else task.val__getitem__(index) + torch.testing.assert_close(second["X"], waveform[2:3]) + assert selections == [1, 2] + continue + + assert not selections + if channel == "all": + expected = waveform + elif channel is None or channel == "downmix": + expected = waveform.mean(dim=0, keepdim=True) + else: + expected = waveform[channel : channel + 1] + torch.testing.assert_close(sample["X"], expected) + torch.testing.assert_close( + task.collate_X([sample, sample]), torch.stack([expected, expected]) + ) + if stage == "development": + torch.testing.assert_close(task.val__getitem__(index)["X"], expected) + assert not selections + + +def make_task(tmp_path, validation_channels=(0,)): + return SpeakerDiarization( + ChannelProtocol(tmp_path / "audio.wav", ("random",), validation_channels), + cache=tmp_path / "task.npz", + duration=1.0, + max_speakers_per_chunk=2, + num_workers=0, + ) + + +def test_channel_balancing_rejected(tmp_path): + task = make_task(tmp_path) + task.prepare_data() + task.setup() + task.model = SimpleSegmentationModel() + task.balance = ["channel"] + with pytest.raises( + ValueError, match="Channel selection cannot be used for balancing" + ): + next(task.train__iter__()) diff --git a/tests/test_channel_policy.py b/tests/test_channel_policy.py new file mode 100644 index 00000000..9bc137e7 --- /dev/null +++ b/tests/test_channel_policy.py @@ -0,0 +1,142 @@ +"""Channel policies behave identically with raw, preloaded, and cached audio.""" + +import random +import wave +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from pyannote.core import Annotation, Segment +from pyannote.database.protocol import SpeakerVerificationProtocol +from pyannote.database.protocol.protocol import ProtocolFile + +from pyannote.audio.core.io import Audio, Channel +from pyannote.audio.core.pipeline import Pipeline +from pyannote.audio.tasks.embedding.mixins import ( + SupervisedRepresentationLearningTaskMixin, +) +from pyannote.audio.utils.preprocessors import Waveform + + +@pytest.fixture +def multichannel_file(tmp_path): + samples = np.tile(np.array([4096, 8192, 24576], dtype="