mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
improve(train): speed up dataloaders
This commit is contained in:
1 parent
d381589409
commit
97f1a1d8ce
4 files changed
+61
-21
No files matched your search
+14
-7
@@ -4,17 +4,14 @@
|
||||
|
||||
### TL;DR
|
||||
|
||||
#### Quality-of-Life improvements
|
||||
#### Faster training
|
||||
|
||||
Models can now be stored alongside their pipelines in the same repository, streamlining gating mechanism:
|
||||
- accept `pyannote/speaker-diarization-x.x` pipeline user agreement
|
||||
- ~~accept `pyannote/segmentation-3.0` model user agreement~~
|
||||
- ~~accept `pyannote/wespeaker-voxceleb-resnet34-LM` model user agreement~~
|
||||
- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=True)`
|
||||
Metadata caching and optimized dataloaders make training on large scale datasets much faster.
|
||||
This led to a 15x speed up on [pyannoteAI](https://www.pyannote.ai) internal large scale training.
|
||||
|
||||
#### [pyannoteAI](https://www.pyannote.ai) premium speaker diarization
|
||||
|
||||
Change one line of code to use [pyannoteAI](https://docs.pyannote.ai) and enjoy **more accurate speaker diarization**.
|
||||
Change one line of code to use [pyannoteAI](https://docs.pyannote.ai) premium models and enjoy **more accurate speaker diarization**.
|
||||
|
||||
```diff
|
||||
from pyannote.audio import Pipeline
|
||||
@@ -24,6 +21,15 @@ pipeline = Pipeline.from_pretrained(
|
||||
diarization = pipeline("/path/to/conversation.wav")
|
||||
```
|
||||
|
||||
#### Quality-of-Life improvements
|
||||
|
||||
Models can now be stored alongside their pipelines in the same repository, streamlining offline use and gating mechanism:
|
||||
- accept `pyannote/speaker-diarization-x.x` pipeline user agreement
|
||||
- ~~accept `pyannote/segmentation-3.0` model user agreement~~
|
||||
- ~~accept `pyannote/wespeaker-voxceleb-resnet34-LM` model user agreement~~
|
||||
- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=True)`
|
||||
|
||||
|
||||
### Breaking changes
|
||||
|
||||
- BREAKING(setup): drop support to `Python` < 3.10
|
||||
@@ -54,6 +60,7 @@ diarization = pipeline("/path/to/conversation.wav")
|
||||
|
||||
- improve(model): improve WavLM (un)freezing support for `SSeRiouSS` architecture ([@clement-pages](https://github.com/clement-pages/))
|
||||
- improve(task): improve `SpeakerDiarization` training with manual optimization ([@clement-pages](https://github.com/clement-pages/))
|
||||
- improve(train): speed up dataloaders
|
||||
- improve(setup): switch to `uv`
|
||||
- improve(setup): switch to `lightning` from `pytorch-lightning`
|
||||
|
||||
|
||||
@@ -323,7 +323,9 @@ class Task(lightning.LightningDataModule):
|
||||
'audio-encoding': array of N audio encodings
|
||||
'audio-annotated': array of N annotated duration (usually equals file duration but might be shorter if file is not fully annotated)
|
||||
'annotations-regions': array of M annotated regions
|
||||
'audio-regions-ids': array of N start/end indices of annotated regions
|
||||
'annotations-segments': array of M' annotated segments
|
||||
'audio-segments-ids': array of N start/end indices of annotated segments
|
||||
'metadata-values': dict of lists of values for subset, scope and database
|
||||
'metadata-`database-name`-labels': array of `database-name` labels. Each database with "database" scope labels has it own array.
|
||||
'metadata-labels': array of global scope labels
|
||||
@@ -356,7 +358,11 @@ class Task(lightning.LightningDataModule):
|
||||
|
||||
annotated_duration = list() # total duration of annotated regions (per file)
|
||||
annotated_regions = list() # annotated regions
|
||||
audio_regions_ids = list() # start/end indices of annotated regions (per file)
|
||||
|
||||
annotations = list() # actual annotations
|
||||
audio_segments_ids = list() # start/end indices of annotated segments (per file)
|
||||
|
||||
unique_labels = list()
|
||||
database_unique_labels = {}
|
||||
|
||||
@@ -368,6 +374,10 @@ class Task(lightning.LightningDataModule):
|
||||
else:
|
||||
files_iter = zip(itertools.repeat("train"), self.protocol.train())
|
||||
|
||||
regions_id: int = 0
|
||||
segments_id: int = 0
|
||||
|
||||
|
||||
for file_id, (subset, file) in enumerate(files_iter):
|
||||
# gather metadata and update metadata_unique_values so that each metadatum
|
||||
# (e.g. source database or label) is represented by an integer.
|
||||
@@ -438,6 +448,8 @@ class Task(lightning.LightningDataModule):
|
||||
)
|
||||
audio_encodings.append(audio_info.encoding) # encoding
|
||||
|
||||
_regions_id = regions_id
|
||||
|
||||
# annotated regions and duration
|
||||
_annotated_duration = 0.0
|
||||
for segment in file["annotated"]:
|
||||
@@ -456,9 +468,17 @@ class Task(lightning.LightningDataModule):
|
||||
# increment annotated duration
|
||||
_annotated_duration += segment.duration
|
||||
|
||||
# increment current regions id
|
||||
regions_id += 1
|
||||
|
||||
# append annotated duration
|
||||
annotated_duration.append(_annotated_duration)
|
||||
|
||||
# append region IDs (start and end time)
|
||||
audio_regions_ids.append((_regions_id, regions_id))
|
||||
|
||||
_segments_id = segments_id
|
||||
|
||||
# annotations
|
||||
for segment, _, label in file["annotation"].itertracks(yield_label=True):
|
||||
# "scope" is provided by speaker diarization protocols to indicate
|
||||
@@ -505,6 +525,12 @@ class Task(lightning.LightningDataModule):
|
||||
)
|
||||
)
|
||||
|
||||
# increment current segment id
|
||||
segments_id += 1
|
||||
|
||||
# append segment IDs (start and end time)
|
||||
audio_segments_ids.append((_segments_id, segments_id))
|
||||
|
||||
# since not all metadata keys are present in all files, fallback to -1 when a key is missing
|
||||
metadata = [
|
||||
tuple(metadatum.get(key, -1) for key in metadata_unique_values)
|
||||
@@ -579,11 +605,21 @@ class Task(lightning.LightningDataModule):
|
||||
)
|
||||
annotated_regions.clear()
|
||||
|
||||
prepared_data["audio-regions-ids"] = np.array(
|
||||
audio_regions_ids, dtype=[("start", "i"), ("end", "i")]
|
||||
)
|
||||
audio_regions_ids.clear()
|
||||
|
||||
prepared_data["annotations-segments"] = np.array(
|
||||
annotations, dtype=segment_dtype
|
||||
)
|
||||
annotations.clear()
|
||||
|
||||
prepared_data["audio-segments-ids"] = np.array(
|
||||
audio_segments_ids, dtype=[("start", "i"), ("end", "i")]
|
||||
)
|
||||
audio_segments_ids.clear()
|
||||
|
||||
prepared_data["metadata-values"] = metadata_unique_values
|
||||
|
||||
for database, labels in database_unique_labels.items():
|
||||
|
||||
@@ -127,28 +127,26 @@ class SegmentationTask(Task):
|
||||
|
||||
# generate `num_chunks_per_file` chunks from this file
|
||||
for _ in range(num_chunks_per_file):
|
||||
# find indices of annotated regions in this file
|
||||
annotated_region_indices = np.where(
|
||||
self.prepared_data["annotations-regions"]["file_id"] == file_id
|
||||
)[0]
|
||||
# read indices of annotated regions in this file
|
||||
start_id, end_id = self.prepared_data["audio-regions-ids"][file_id]
|
||||
|
||||
# turn annotated regions duration into a probability distribution
|
||||
|
||||
cum_prob_annotated_regions_duration = np.cumsum(
|
||||
self.prepared_data["annotations-regions"]["duration"][
|
||||
annotated_region_indices
|
||||
start_id:end_id
|
||||
]
|
||||
/ np.sum(
|
||||
self.prepared_data["annotations-regions"]["duration"][
|
||||
annotated_region_indices
|
||||
start_id:end_id
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
# selected one annotated region at random (with probability proportional to its duration)
|
||||
annotated_region_index = annotated_region_indices[
|
||||
cum_prob_annotated_regions_duration.searchsorted(rng.random())
|
||||
]
|
||||
annotated_region_index = (
|
||||
start_id
|
||||
+ cum_prob_annotated_regions_duration.searchsorted(rng.random())
|
||||
)
|
||||
|
||||
# select one chunk at random in this annotated region
|
||||
_, region_duration, start = self.prepared_data["annotations-regions"][
|
||||
|
||||
@@ -257,7 +257,7 @@ class SpeakerDiarization(SegmentationTask):
|
||||
resolution=Resolution.FRAME,
|
||||
duration=self.duration,
|
||||
min_duration=self.min_duration,
|
||||
classes=[f"speaker#{i+1}" for i in range(self.max_speakers_per_chunk)],
|
||||
classes=[f"speaker#{i + 1}" for i in range(self.max_speakers_per_chunk)],
|
||||
powerset_max_classes=self.max_speakers_per_frame,
|
||||
permutation_invariant=True,
|
||||
)
|
||||
@@ -306,9 +306,8 @@ class SpeakerDiarization(SegmentationTask):
|
||||
sample["X"], _ = self.model.audio.crop(file, chunk, duration=duration)
|
||||
|
||||
# gather all annotations of current file
|
||||
annotations = self.prepared_data["annotations-segments"][
|
||||
self.prepared_data["annotations-segments"]["file_id"] == file_id
|
||||
]
|
||||
start_id, end_id = self.prepared_data["audio-segments-ids"][file_id]
|
||||
annotations = self.prepared_data["annotations-segments"][start_id:end_id]
|
||||
|
||||
# gather all annotations with non-empty intersection with current chunk
|
||||
chunk_annotations = annotations[
|
||||
|
||||
Reference in new issue
Block a user