improve(train): speed up dataloaders

This commit is contained in:
Hervé BREDIN authored and GitHub committed 2025-03-28 11:06:21 +01:00
1 parent d381589409
commit 97f1a1d8ce
4 files changed
+61 -21

No files matched your search

+14 -7
View File
@@ -4,17 +4,14 @@
### TL;DR
#### Quality-of-Life improvements
#### Faster training
Models can now be stored alongside their pipelines in the same repository, streamlining gating mechanism:
- accept `pyannote/speaker-diarization-x.x` pipeline user agreement
- ~~accept `pyannote/segmentation-3.0` model user agreement~~
- ~~accept `pyannote/wespeaker-voxceleb-resnet34-LM` model user agreement~~
- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=True)`
Metadata caching and optimized dataloaders make training on large scale datasets much faster.
This led to a 15x speed up on [pyannoteAI](https://www.pyannote.ai) internal large scale training.
#### [pyannoteAI](https://www.pyannote.ai) premium speaker diarization
Change one line of code to use [pyannoteAI](https://docs.pyannote.ai) and enjoy **more accurate speaker diarization**.
Change one line of code to use [pyannoteAI](https://docs.pyannote.ai) premium models and enjoy **more accurate speaker diarization**.
```diff
from pyannote.audio import Pipeline
@@ -24,6 +21,15 @@ pipeline = Pipeline.from_pretrained(
diarization = pipeline("/path/to/conversation.wav")
```
#### Quality-of-Life improvements
Models can now be stored alongside their pipelines in the same repository, streamlining offline use and gating mechanism:
- accept `pyannote/speaker-diarization-x.x` pipeline user agreement
- ~~accept `pyannote/segmentation-3.0` model user agreement~~
- ~~accept `pyannote/wespeaker-voxceleb-resnet34-LM` model user agreement~~
- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=True)`
### Breaking changes
- BREAKING(setup): drop support to `Python` < 3.10
@@ -54,6 +60,7 @@ diarization = pipeline("/path/to/conversation.wav")
- improve(model): improve WavLM (un)freezing support for `SSeRiouSS` architecture ([@clement-pages](https://github.com/clement-pages/))
- improve(task): improve `SpeakerDiarization` training with manual optimization ([@clement-pages](https://github.com/clement-pages/))
- improve(train): speed up dataloaders
- improve(setup): switch to `uv`
- improve(setup): switch to `lightning` from `pytorch-lightning`
+36
View File
@@ -323,7 +323,9 @@ class Task(lightning.LightningDataModule):
'audio-encoding': array of N audio encodings
'audio-annotated': array of N annotated duration (usually equals file duration but might be shorter if file is not fully annotated)
'annotations-regions': array of M annotated regions
'audio-regions-ids': array of N start/end indices of annotated regions
'annotations-segments': array of M' annotated segments
'audio-segments-ids': array of N start/end indices of annotated segments
'metadata-values': dict of lists of values for subset, scope and database
'metadata-`database-name`-labels': array of `database-name` labels. Each database with "database" scope labels has it own array.
'metadata-labels': array of global scope labels
@@ -356,7 +358,11 @@ class Task(lightning.LightningDataModule):
annotated_duration = list() # total duration of annotated regions (per file)
annotated_regions = list() # annotated regions
audio_regions_ids = list() # start/end indices of annotated regions (per file)
annotations = list() # actual annotations
audio_segments_ids = list() # start/end indices of annotated segments (per file)
unique_labels = list()
database_unique_labels = {}
@@ -368,6 +374,10 @@ class Task(lightning.LightningDataModule):
else:
files_iter = zip(itertools.repeat("train"), self.protocol.train())
regions_id: int = 0
segments_id: int = 0
for file_id, (subset, file) in enumerate(files_iter):
# gather metadata and update metadata_unique_values so that each metadatum
# (e.g. source database or label) is represented by an integer.
@@ -438,6 +448,8 @@ class Task(lightning.LightningDataModule):
)
audio_encodings.append(audio_info.encoding) # encoding
_regions_id = regions_id
# annotated regions and duration
_annotated_duration = 0.0
for segment in file["annotated"]:
@@ -456,9 +468,17 @@ class Task(lightning.LightningDataModule):
# increment annotated duration
_annotated_duration += segment.duration
# increment current regions id
regions_id += 1
# append annotated duration
annotated_duration.append(_annotated_duration)
# append region IDs (start and end time)
audio_regions_ids.append((_regions_id, regions_id))
_segments_id = segments_id
# annotations
for segment, _, label in file["annotation"].itertracks(yield_label=True):
# "scope" is provided by speaker diarization protocols to indicate
@@ -505,6 +525,12 @@ class Task(lightning.LightningDataModule):
)
)
# increment current segment id
segments_id += 1
# append segment IDs (start and end time)
audio_segments_ids.append((_segments_id, segments_id))
# since not all metadata keys are present in all files, fallback to -1 when a key is missing
metadata = [
tuple(metadatum.get(key, -1) for key in metadata_unique_values)
@@ -579,11 +605,21 @@ class Task(lightning.LightningDataModule):
)
annotated_regions.clear()
prepared_data["audio-regions-ids"] = np.array(
audio_regions_ids, dtype=[("start", "i"), ("end", "i")]
)
audio_regions_ids.clear()
prepared_data["annotations-segments"] = np.array(
annotations, dtype=segment_dtype
)
annotations.clear()
prepared_data["audio-segments-ids"] = np.array(
audio_segments_ids, dtype=[("start", "i"), ("end", "i")]
)
audio_segments_ids.clear()
prepared_data["metadata-values"] = metadata_unique_values
for database, labels in database_unique_labels.items():
@@ -127,28 +127,26 @@ class SegmentationTask(Task):
# generate `num_chunks_per_file` chunks from this file
for _ in range(num_chunks_per_file):
# find indices of annotated regions in this file
annotated_region_indices = np.where(
self.prepared_data["annotations-regions"]["file_id"] == file_id
)[0]
# read indices of annotated regions in this file
start_id, end_id = self.prepared_data["audio-regions-ids"][file_id]
# turn annotated regions duration into a probability distribution
cum_prob_annotated_regions_duration = np.cumsum(
self.prepared_data["annotations-regions"]["duration"][
annotated_region_indices
start_id:end_id
]
/ np.sum(
self.prepared_data["annotations-regions"]["duration"][
annotated_region_indices
start_id:end_id
]
)
)
# selected one annotated region at random (with probability proportional to its duration)
annotated_region_index = annotated_region_indices[
cum_prob_annotated_regions_duration.searchsorted(rng.random())
]
annotated_region_index = (
start_id
+ cum_prob_annotated_regions_duration.searchsorted(rng.random())
)
# select one chunk at random in this annotated region
_, region_duration, start = self.prepared_data["annotations-regions"][
@@ -257,7 +257,7 @@ class SpeakerDiarization(SegmentationTask):
resolution=Resolution.FRAME,
duration=self.duration,
min_duration=self.min_duration,
classes=[f"speaker#{i+1}" for i in range(self.max_speakers_per_chunk)],
classes=[f"speaker#{i + 1}" for i in range(self.max_speakers_per_chunk)],
powerset_max_classes=self.max_speakers_per_frame,
permutation_invariant=True,
)
@@ -306,9 +306,8 @@ class SpeakerDiarization(SegmentationTask):
sample["X"], _ = self.model.audio.crop(file, chunk, duration=duration)
# gather all annotations of current file
annotations = self.prepared_data["annotations-segments"][
self.prepared_data["annotations-segments"]["file_id"] == file_id
]
start_id, end_id = self.prepared_data["audio-segments-ids"][file_id]
annotations = self.prepared_data["annotations-segments"][start_id:end_id]
# gather all annotations with non-empty intersection with current chunk
chunk_annotations = annotations[