fix(task): fix random generators (#1594)

Before this change, each worker would select the same files, resulting in less randomness than expected.
This commit is contained in:
Hervé BREDIN authored and GitHub committed 2024-01-09 13:06:09 +01:00
1 parent eda0c51b13
commit 8f477fada9
5 files changed
+37 -21

No files matched your search

+4
View File
@@ -7,6 +7,10 @@
- feat(pipeline): add `Waveform` and `SampleRate` preprocessors
- feat(model): add `num_frames` and `receptive_field` to segmentation models
### Fixes
- fix(task): fix random generators
## Version 3.1.1 (2023-12-01)
### TL;DR
+1 -1
View File
@@ -145,7 +145,7 @@ class SupervisedRepresentationLearningTaskMixin:
"""
# create worker-specific random number generator
rng = create_rng_for_worker(self.model.current_epoch)
rng = create_rng_for_worker(self.model)
classes = list(self.specifications.classes)
+17 -10
View File
@@ -421,12 +421,16 @@ class SegmentationTaskMixin:
# indices of training files that matches domain filters
training = self.metadata["subset"] == Subsets.index("train")
for key, value in filters.items():
training &= self.metadata[key] == self.metadata_unique_values[key].index(value)
training &= self.metadata[key] == self.metadata_unique_values[key].index(
value
)
file_ids = np.where(training)[0]
# turn annotated duration into a probability distribution
annotated_duration = self.annotated_duration[file_ids]
prob_annotated_duration = annotated_duration / np.sum(annotated_duration)
cum_prob_annotated_duration = np.cumsum(
annotated_duration / np.sum(annotated_duration)
)
duration = self.duration
@@ -434,7 +438,7 @@ class SegmentationTaskMixin:
while True:
# select one file at random (with probability proportional to its annotated duration)
file_id = np.random.choice(file_ids, p=prob_annotated_duration)
file_id = file_ids[cum_prob_annotated_duration.searchsorted(rng.random())]
# generate `num_chunks_per_file` chunks from this file
for _ in range(num_chunks_per_file):
@@ -444,14 +448,17 @@ class SegmentationTaskMixin:
)[0]
# turn annotated regions duration into a probability distribution
prob_annotated_regions_duration = self.annotated_regions["duration"][
annotated_region_indices
] / np.sum(self.annotated_regions["duration"][annotated_region_indices])
cum_prob_annotated_regions_duration = np.cumsum(
self.annotated_regions["duration"][annotated_region_indices]
/ np.sum(
self.annotated_regions["duration"][annotated_region_indices]
)
)
# selected one annotated region at random (with probability proportional to its duration)
annotated_region_index = np.random.choice(
annotated_region_indices, p=prob_annotated_regions_duration
)
annotated_region_index = annotated_region_indices[
cum_prob_annotated_regions_duration.searchsorted(rng.random())
]
# select one chunk at random in this annotated region
_, _, start, end = self.annotated_regions[annotated_region_index]
@@ -475,7 +482,7 @@ class SegmentationTaskMixin:
"""
# create worker-specific random number generator
rng = create_rng_for_worker(self.model.current_epoch)
rng = create_rng_for_worker(self.model)
balance = getattr(self, "balance", None)
if balance is None:
+14 -9
View File
@@ -27,7 +27,7 @@ from random import Random
import torch
def create_rng_for_worker(epoch: int) -> Random:
def create_rng_for_worker(model) -> Random:
"""Create worker-specific random number generator
This makes sure that
@@ -43,19 +43,24 @@ def create_rng_for_worker(epoch: int) -> Random:
# create random number generator
rng = Random()
#  create seed as a combination of PL_GLOBAL_SEED (set by pl.seed_everything())
#  and other PL multi-processing variables
global_seed = int(os.environ.get("PL_GLOBAL_SEED", "0"))
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
node_rank = int(os.environ.get("NODE_RANK", "0"))
global_seed = os.environ.get("PL_GLOBAL_SEED", "unset")
worker_info = torch.utils.data.get_worker_info()
if worker_info is None:
worker_id = 0
worker_id = None
else:
worker_id = worker_info.id
rng.seed(hash((global_seed, worker_id, local_rank, node_rank, epoch)))
seed = hash(
(
global_seed,
worker_id,
model.local_rank,
model.global_rank,
model.current_epoch,
)
)
rng.seed(seed)
return rng
+1 -1
View File
@@ -236,7 +236,7 @@
"\n",
" # create worker-specific random number generator (RNG) to avoid this common bug:\n",
" # tanelp.github.io/posts/a-bug-that-plagues-thousands-of-open-source-ml-projects/\n",
" rng = create_rng_for_worker(self.model.current_epoch)\n",
" rng = create_rng_for_worker(self.model)\n",
"\n",
" # load list and number of classes\n",
" classes = self.specifications.classes\n",