diff --git a/CHANGELOG.md b/CHANGELOG.md index d663f452..536190e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,17 +4,14 @@ ### TL;DR -#### Quality-of-Life improvements +#### Faster training -Models can now be stored alongside their pipelines in the same repository, streamlining gating mechanism: -- accept `pyannote/speaker-diarization-x.x` pipeline user agreement -- ~~accept `pyannote/segmentation-3.0` model user agreement~~ -- ~~accept `pyannote/wespeaker-voxceleb-resnet34-LM` model user agreement~~ -- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=True)` +Metadata caching and optimized dataloaders make training on large scale datasets much faster. +This led to a 15x speed up on [pyannoteAI](https://www.pyannote.ai) internal large scale training. #### [pyannoteAI](https://www.pyannote.ai) premium speaker diarization -Change one line of code to use [pyannoteAI](https://docs.pyannote.ai) and enjoy **more accurate speaker diarization**. +Change one line of code to use [pyannoteAI](https://docs.pyannote.ai) premium models and enjoy **more accurate speaker diarization**. ```diff from pyannote.audio import Pipeline @@ -24,6 +21,15 @@ pipeline = Pipeline.from_pretrained( diarization = pipeline("/path/to/conversation.wav") ``` +#### Quality-of-Life improvements + +Models can now be stored alongside their pipelines in the same repository, streamlining offline use and gating mechanism: +- accept `pyannote/speaker-diarization-x.x` pipeline user agreement +- ~~accept `pyannote/segmentation-3.0` model user agreement~~ +- ~~accept `pyannote/wespeaker-voxceleb-resnet34-LM` model user agreement~~ +- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=True)` + + ### Breaking changes - BREAKING(setup): drop support to `Python` < 3.10 @@ -54,6 +60,7 @@ diarization = pipeline("/path/to/conversation.wav") - improve(model): improve WavLM (un)freezing support for `SSeRiouSS` architecture ([@clement-pages](https://github.com/clement-pages/)) - improve(task): improve `SpeakerDiarization` training with manual optimization ([@clement-pages](https://github.com/clement-pages/)) +- improve(train): speed up dataloaders - improve(setup): switch to `uv` - improve(setup): switch to `lightning` from `pytorch-lightning` diff --git a/src/pyannote/audio/core/task.py b/src/pyannote/audio/core/task.py index fef4dbeb..4aeaaa32 100644 --- a/src/pyannote/audio/core/task.py +++ b/src/pyannote/audio/core/task.py @@ -323,7 +323,9 @@ class Task(lightning.LightningDataModule): 'audio-encoding': array of N audio encodings 'audio-annotated': array of N annotated duration (usually equals file duration but might be shorter if file is not fully annotated) 'annotations-regions': array of M annotated regions + 'audio-regions-ids': array of N start/end indices of annotated regions 'annotations-segments': array of M' annotated segments + 'audio-segments-ids': array of N start/end indices of annotated segments 'metadata-values': dict of lists of values for subset, scope and database 'metadata-`database-name`-labels': array of `database-name` labels. Each database with "database" scope labels has it own array. 'metadata-labels': array of global scope labels @@ -356,7 +358,11 @@ class Task(lightning.LightningDataModule): annotated_duration = list() # total duration of annotated regions (per file) annotated_regions = list() # annotated regions + audio_regions_ids = list() # start/end indices of annotated regions (per file) + annotations = list() # actual annotations + audio_segments_ids = list() # start/end indices of annotated segments (per file) + unique_labels = list() database_unique_labels = {} @@ -368,6 +374,10 @@ class Task(lightning.LightningDataModule): else: files_iter = zip(itertools.repeat("train"), self.protocol.train()) + regions_id: int = 0 + segments_id: int = 0 + + for file_id, (subset, file) in enumerate(files_iter): # gather metadata and update metadata_unique_values so that each metadatum # (e.g. source database or label) is represented by an integer. @@ -438,6 +448,8 @@ class Task(lightning.LightningDataModule): ) audio_encodings.append(audio_info.encoding) # encoding + _regions_id = regions_id + # annotated regions and duration _annotated_duration = 0.0 for segment in file["annotated"]: @@ -456,9 +468,17 @@ class Task(lightning.LightningDataModule): # increment annotated duration _annotated_duration += segment.duration + # increment current regions id + regions_id += 1 + # append annotated duration annotated_duration.append(_annotated_duration) + # append region IDs (start and end time) + audio_regions_ids.append((_regions_id, regions_id)) + + _segments_id = segments_id + # annotations for segment, _, label in file["annotation"].itertracks(yield_label=True): # "scope" is provided by speaker diarization protocols to indicate @@ -505,6 +525,12 @@ class Task(lightning.LightningDataModule): ) ) + # increment current segment id + segments_id += 1 + + # append segment IDs (start and end time) + audio_segments_ids.append((_segments_id, segments_id)) + # since not all metadata keys are present in all files, fallback to -1 when a key is missing metadata = [ tuple(metadatum.get(key, -1) for key in metadata_unique_values) @@ -579,11 +605,21 @@ class Task(lightning.LightningDataModule): ) annotated_regions.clear() + prepared_data["audio-regions-ids"] = np.array( + audio_regions_ids, dtype=[("start", "i"), ("end", "i")] + ) + audio_regions_ids.clear() + prepared_data["annotations-segments"] = np.array( annotations, dtype=segment_dtype ) annotations.clear() + prepared_data["audio-segments-ids"] = np.array( + audio_segments_ids, dtype=[("start", "i"), ("end", "i")] + ) + audio_segments_ids.clear() + prepared_data["metadata-values"] = metadata_unique_values for database, labels in database_unique_labels.items(): diff --git a/src/pyannote/audio/tasks/segmentation/mixins.py b/src/pyannote/audio/tasks/segmentation/mixins.py index d8047c2f..db6b1f09 100644 --- a/src/pyannote/audio/tasks/segmentation/mixins.py +++ b/src/pyannote/audio/tasks/segmentation/mixins.py @@ -127,28 +127,26 @@ class SegmentationTask(Task): # generate `num_chunks_per_file` chunks from this file for _ in range(num_chunks_per_file): - # find indices of annotated regions in this file - annotated_region_indices = np.where( - self.prepared_data["annotations-regions"]["file_id"] == file_id - )[0] + # read indices of annotated regions in this file + start_id, end_id = self.prepared_data["audio-regions-ids"][file_id] # turn annotated regions duration into a probability distribution - cum_prob_annotated_regions_duration = np.cumsum( self.prepared_data["annotations-regions"]["duration"][ - annotated_region_indices + start_id:end_id ] / np.sum( self.prepared_data["annotations-regions"]["duration"][ - annotated_region_indices + start_id:end_id ] ) ) # selected one annotated region at random (with probability proportional to its duration) - annotated_region_index = annotated_region_indices[ - cum_prob_annotated_regions_duration.searchsorted(rng.random()) - ] + annotated_region_index = ( + start_id + + cum_prob_annotated_regions_duration.searchsorted(rng.random()) + ) # select one chunk at random in this annotated region _, region_duration, start = self.prepared_data["annotations-regions"][ diff --git a/src/pyannote/audio/tasks/segmentation/speaker_diarization.py b/src/pyannote/audio/tasks/segmentation/speaker_diarization.py index 40890711..09eca916 100644 --- a/src/pyannote/audio/tasks/segmentation/speaker_diarization.py +++ b/src/pyannote/audio/tasks/segmentation/speaker_diarization.py @@ -257,7 +257,7 @@ class SpeakerDiarization(SegmentationTask): resolution=Resolution.FRAME, duration=self.duration, min_duration=self.min_duration, - classes=[f"speaker#{i+1}" for i in range(self.max_speakers_per_chunk)], + classes=[f"speaker#{i + 1}" for i in range(self.max_speakers_per_chunk)], powerset_max_classes=self.max_speakers_per_frame, permutation_invariant=True, ) @@ -306,9 +306,8 @@ class SpeakerDiarization(SegmentationTask): sample["X"], _ = self.model.audio.crop(file, chunk, duration=duration) # gather all annotations of current file - annotations = self.prepared_data["annotations-segments"][ - self.prepared_data["annotations-segments"]["file_id"] == file_id - ] + start_id, end_id = self.prepared_data["audio-segments-ids"][file_id] + annotations = self.prepared_data["annotations-segments"][start_id:end_id] # gather all annotations with non-empty intersection with current chunk chunk_annotations = annotations[