fix: create rng worker same data each epoch #526 (#527)

* fix: fix #526

* wip: better seeding

* fix: ensure that seeds are different for each epoch

* fix: set and use Task.current_epoch

* fix: set default current_epoch to 0

Co-authored-by: Harry Coultas Blum <himion0@gmail.com>
This commit is contained in:
Hervé BREDINandHarry Coultas Blum authored and GitHub committed 2020-12-02 10:48:54 +00:00
1 parent 3406aa88d3
commit 889f2b2ed7
7 files changed
+47 -12

No files matched your search

+3
View File
@@ -421,6 +421,9 @@ class Model(pl.LightningModule):
return self.helper_default_activation(task_specifications)
def on_epoch_start(self):
self.task.current_epoch = self.current_epoch
# training step logic is delegated to the task because the
# model does not really need to know how it is being used.
def training_step(self, batch, batch_idx):
+8
View File
@@ -236,6 +236,14 @@ class Task(pl.LightningDataModule):
""""Check whether multiple tasks are addressed at once"""
return len(self.specifications) > 1
@property
def current_epoch(self) -> int:
return getattr(self, "current_epoch_", 0)
@current_epoch.setter
def current_epoch(self, current_epoch: int):
self.current_epoch_ = current_epoch
def train__iter__(self):
# will become train_dataset.__iter__ method
msg = f"Missing '{self.__class__.__name__}.train__iter__' method."
+2 -1
View File
@@ -173,8 +173,9 @@ class SegmentationTaskMixin:
`frame` is infered automagically from the
example model output.
"""
# create worker-specific random number generator
rng = create_rng_for_worker()
rng = create_rng_for_worker(self.current_epoch)
while True:
@@ -245,7 +245,7 @@ class MultiTaskSegmentation(SegmentationTaskMixin, Task):
"""
# create worker-specific random number generator
rng = create_rng_for_worker()
rng = create_rng_for_worker(self.current_epoch)
if self.osd and self.tasks["osd"].domain is not None:
chunks_by_domain = {
@@ -200,7 +200,7 @@ class OverlappedSpeechDetection(SegmentationTaskMixin, Task):
"""
# create worker-specific random number generator
rng = create_rng_for_worker()
rng = create_rng_for_worker(self.current_epoch)
if self.domain is None:
chunks = self.train__iter__helper(rng)
@@ -158,7 +158,7 @@ class SpeakerEmbeddingArcFace(Task):
"""
# create worker-specific random number generator
rng = create_rng_for_worker()
rng = create_rng_for_worker(self.current_epoch)
speakers = list(self.speakers)
+31 -8
View File
@@ -22,29 +22,52 @@
import os
import random
from random import Random
import torch
def create_rng_for_worker():
def create_rng_for_worker(epoch: int) -> Random:
"""Create worker-specific random number generator
This makes sure that
1. training samples generation is reproducible
2. all workers use a different seed (e.g. to avoid training samples duplication)
2. every (worker, epoch) uses a different seed
Parameters
----------
epoch : int
Current epoch.
"""
# create random number generator
rng = random.Random()
rng = Random()
#  create seed as a combination of PL_GLOBAL_SEED (set by pl.seed_everything())
#  and current worker id.
global_seed = int(os.environ.get("PL_GLOBAL_SEED", 1))
#  and other PL multi-processing variables
global_seed = int(os.environ.get("PL_GLOBAL_SEED", "0"))
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
node_rank = int(os.environ.get("NODE_RANK", "0"))
num_gpus = len(os.environ.get("PL_TRAINER_GPUS", "0").split(","))
world_size = int(os.environ.get("WORLD_SIZE", "1"))
worker_info = torch.utils.data.get_worker_info()
if worker_info is None:
rng.seed(global_seed)
num_workers = 1
worker_id = 0
else:
rng.seed(global_seed + worker_info.id)
num_workers = worker_info.num_workers
worker_id = worker_info.id
seed = (
global_seed
+ worker_id
+ local_rank * num_workers
+ node_rank * num_workers * num_gpus
+ epoch * num_workers * world_size
)
rng.seed(seed)
return rng