mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
fix(task): fix random generators (#1594)
Before this change, each worker would select the same files, resulting in less randomness than expected.
This commit is contained in:
1 parent
eda0c51b13
commit
8f477fada9
5 files changed
+37
-21
No files matched your search
@@ -7,6 +7,10 @@
|
||||
- feat(pipeline): add `Waveform` and `SampleRate` preprocessors
|
||||
- feat(model): add `num_frames` and `receptive_field` to segmentation models
|
||||
|
||||
### Fixes
|
||||
|
||||
- fix(task): fix random generators
|
||||
|
||||
## Version 3.1.1 (2023-12-01)
|
||||
|
||||
### TL;DR
|
||||
|
||||
@@ -145,7 +145,7 @@ class SupervisedRepresentationLearningTaskMixin:
|
||||
"""
|
||||
|
||||
# create worker-specific random number generator
|
||||
rng = create_rng_for_worker(self.model.current_epoch)
|
||||
rng = create_rng_for_worker(self.model)
|
||||
|
||||
classes = list(self.specifications.classes)
|
||||
|
||||
|
||||
@@ -421,12 +421,16 @@ class SegmentationTaskMixin:
|
||||
# indices of training files that matches domain filters
|
||||
training = self.metadata["subset"] == Subsets.index("train")
|
||||
for key, value in filters.items():
|
||||
training &= self.metadata[key] == self.metadata_unique_values[key].index(value)
|
||||
training &= self.metadata[key] == self.metadata_unique_values[key].index(
|
||||
value
|
||||
)
|
||||
file_ids = np.where(training)[0]
|
||||
|
||||
# turn annotated duration into a probability distribution
|
||||
annotated_duration = self.annotated_duration[file_ids]
|
||||
prob_annotated_duration = annotated_duration / np.sum(annotated_duration)
|
||||
cum_prob_annotated_duration = np.cumsum(
|
||||
annotated_duration / np.sum(annotated_duration)
|
||||
)
|
||||
|
||||
duration = self.duration
|
||||
|
||||
@@ -434,7 +438,7 @@ class SegmentationTaskMixin:
|
||||
|
||||
while True:
|
||||
# select one file at random (with probability proportional to its annotated duration)
|
||||
file_id = np.random.choice(file_ids, p=prob_annotated_duration)
|
||||
file_id = file_ids[cum_prob_annotated_duration.searchsorted(rng.random())]
|
||||
|
||||
# generate `num_chunks_per_file` chunks from this file
|
||||
for _ in range(num_chunks_per_file):
|
||||
@@ -444,14 +448,17 @@ class SegmentationTaskMixin:
|
||||
)[0]
|
||||
|
||||
# turn annotated regions duration into a probability distribution
|
||||
prob_annotated_regions_duration = self.annotated_regions["duration"][
|
||||
annotated_region_indices
|
||||
] / np.sum(self.annotated_regions["duration"][annotated_region_indices])
|
||||
cum_prob_annotated_regions_duration = np.cumsum(
|
||||
self.annotated_regions["duration"][annotated_region_indices]
|
||||
/ np.sum(
|
||||
self.annotated_regions["duration"][annotated_region_indices]
|
||||
)
|
||||
)
|
||||
|
||||
# selected one annotated region at random (with probability proportional to its duration)
|
||||
annotated_region_index = np.random.choice(
|
||||
annotated_region_indices, p=prob_annotated_regions_duration
|
||||
)
|
||||
annotated_region_index = annotated_region_indices[
|
||||
cum_prob_annotated_regions_duration.searchsorted(rng.random())
|
||||
]
|
||||
|
||||
# select one chunk at random in this annotated region
|
||||
_, _, start, end = self.annotated_regions[annotated_region_index]
|
||||
@@ -475,7 +482,7 @@ class SegmentationTaskMixin:
|
||||
"""
|
||||
|
||||
# create worker-specific random number generator
|
||||
rng = create_rng_for_worker(self.model.current_epoch)
|
||||
rng = create_rng_for_worker(self.model)
|
||||
|
||||
balance = getattr(self, "balance", None)
|
||||
if balance is None:
|
||||
|
||||
@@ -27,7 +27,7 @@ from random import Random
|
||||
import torch
|
||||
|
||||
|
||||
def create_rng_for_worker(epoch: int) -> Random:
|
||||
def create_rng_for_worker(model) -> Random:
|
||||
"""Create worker-specific random number generator
|
||||
|
||||
This makes sure that
|
||||
@@ -43,19 +43,24 @@ def create_rng_for_worker(epoch: int) -> Random:
|
||||
# create random number generator
|
||||
rng = Random()
|
||||
|
||||
# create seed as a combination of PL_GLOBAL_SEED (set by pl.seed_everything())
|
||||
# and other PL multi-processing variables
|
||||
global_seed = int(os.environ.get("PL_GLOBAL_SEED", "0"))
|
||||
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
|
||||
node_rank = int(os.environ.get("NODE_RANK", "0"))
|
||||
|
||||
global_seed = os.environ.get("PL_GLOBAL_SEED", "unset")
|
||||
worker_info = torch.utils.data.get_worker_info()
|
||||
|
||||
if worker_info is None:
|
||||
worker_id = 0
|
||||
worker_id = None
|
||||
else:
|
||||
worker_id = worker_info.id
|
||||
|
||||
rng.seed(hash((global_seed, worker_id, local_rank, node_rank, epoch)))
|
||||
seed = hash(
|
||||
(
|
||||
global_seed,
|
||||
worker_id,
|
||||
model.local_rank,
|
||||
model.global_rank,
|
||||
model.current_epoch,
|
||||
)
|
||||
)
|
||||
|
||||
rng.seed(seed)
|
||||
|
||||
return rng
|
||||
@@ -236,7 +236,7 @@
|
||||
"\n",
|
||||
" # create worker-specific random number generator (RNG) to avoid this common bug:\n",
|
||||
" # tanelp.github.io/posts/a-bug-that-plagues-thousands-of-open-source-ml-projects/\n",
|
||||
" rng = create_rng_for_worker(self.model.current_epoch)\n",
|
||||
" rng = create_rng_for_worker(self.model)\n",
|
||||
"\n",
|
||||
" # load list and number of classes\n",
|
||||
" classes = self.specifications.classes\n",
|
||||
|
||||
Reference in new issue
Block a user