mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
* fix: fix #526 * wip: better seeding * fix: ensure that seeds are different for each epoch * fix: set and use Task.current_epoch * fix: set default current_epoch to 0 Co-authored-by: Harry Coultas Blum <himion0@gmail.com>
This commit is contained in:
1 parent
3406aa88d3
commit
889f2b2ed7
7 files changed
+47
-12
No files matched your search
@@ -421,6 +421,9 @@ class Model(pl.LightningModule):
|
||||
|
||||
return self.helper_default_activation(task_specifications)
|
||||
|
||||
def on_epoch_start(self):
|
||||
self.task.current_epoch = self.current_epoch
|
||||
|
||||
# training step logic is delegated to the task because the
|
||||
# model does not really need to know how it is being used.
|
||||
def training_step(self, batch, batch_idx):
|
||||
|
||||
@@ -236,6 +236,14 @@ class Task(pl.LightningDataModule):
|
||||
""""Check whether multiple tasks are addressed at once"""
|
||||
return len(self.specifications) > 1
|
||||
|
||||
@property
|
||||
def current_epoch(self) -> int:
|
||||
return getattr(self, "current_epoch_", 0)
|
||||
|
||||
@current_epoch.setter
|
||||
def current_epoch(self, current_epoch: int):
|
||||
self.current_epoch_ = current_epoch
|
||||
|
||||
def train__iter__(self):
|
||||
# will become train_dataset.__iter__ method
|
||||
msg = f"Missing '{self.__class__.__name__}.train__iter__' method."
|
||||
|
||||
@@ -173,8 +173,9 @@ class SegmentationTaskMixin:
|
||||
`frame` is infered automagically from the
|
||||
example model output.
|
||||
"""
|
||||
|
||||
# create worker-specific random number generator
|
||||
rng = create_rng_for_worker()
|
||||
rng = create_rng_for_worker(self.current_epoch)
|
||||
|
||||
while True:
|
||||
|
||||
|
||||
@@ -245,7 +245,7 @@ class MultiTaskSegmentation(SegmentationTaskMixin, Task):
|
||||
"""
|
||||
|
||||
# create worker-specific random number generator
|
||||
rng = create_rng_for_worker()
|
||||
rng = create_rng_for_worker(self.current_epoch)
|
||||
|
||||
if self.osd and self.tasks["osd"].domain is not None:
|
||||
chunks_by_domain = {
|
||||
|
||||
@@ -200,7 +200,7 @@ class OverlappedSpeechDetection(SegmentationTaskMixin, Task):
|
||||
"""
|
||||
|
||||
# create worker-specific random number generator
|
||||
rng = create_rng_for_worker()
|
||||
rng = create_rng_for_worker(self.current_epoch)
|
||||
|
||||
if self.domain is None:
|
||||
chunks = self.train__iter__helper(rng)
|
||||
|
||||
@@ -158,7 +158,7 @@ class SpeakerEmbeddingArcFace(Task):
|
||||
"""
|
||||
|
||||
# create worker-specific random number generator
|
||||
rng = create_rng_for_worker()
|
||||
rng = create_rng_for_worker(self.current_epoch)
|
||||
|
||||
speakers = list(self.speakers)
|
||||
|
||||
|
||||
@@ -22,29 +22,52 @@
|
||||
|
||||
|
||||
import os
|
||||
import random
|
||||
from random import Random
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def create_rng_for_worker():
|
||||
def create_rng_for_worker(epoch: int) -> Random:
|
||||
"""Create worker-specific random number generator
|
||||
|
||||
This makes sure that
|
||||
1. training samples generation is reproducible
|
||||
2. all workers use a different seed (e.g. to avoid training samples duplication)
|
||||
2. every (worker, epoch) uses a different seed
|
||||
|
||||
Parameters
|
||||
----------
|
||||
epoch : int
|
||||
Current epoch.
|
||||
"""
|
||||
|
||||
# create random number generator
|
||||
rng = random.Random()
|
||||
rng = Random()
|
||||
|
||||
# create seed as a combination of PL_GLOBAL_SEED (set by pl.seed_everything())
|
||||
# and current worker id.
|
||||
global_seed = int(os.environ.get("PL_GLOBAL_SEED", 1))
|
||||
# and other PL multi-processing variables
|
||||
global_seed = int(os.environ.get("PL_GLOBAL_SEED", "0"))
|
||||
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
|
||||
node_rank = int(os.environ.get("NODE_RANK", "0"))
|
||||
num_gpus = len(os.environ.get("PL_TRAINER_GPUS", "0").split(","))
|
||||
world_size = int(os.environ.get("WORLD_SIZE", "1"))
|
||||
|
||||
worker_info = torch.utils.data.get_worker_info()
|
||||
|
||||
if worker_info is None:
|
||||
rng.seed(global_seed)
|
||||
num_workers = 1
|
||||
worker_id = 0
|
||||
else:
|
||||
rng.seed(global_seed + worker_info.id)
|
||||
num_workers = worker_info.num_workers
|
||||
worker_id = worker_info.id
|
||||
|
||||
seed = (
|
||||
global_seed
|
||||
+ worker_id
|
||||
+ local_rank * num_workers
|
||||
+ node_rank * num_workers * num_gpus
|
||||
+ epoch * num_workers * world_size
|
||||
)
|
||||
|
||||
rng.seed(seed)
|
||||
|
||||
return rng
|
||||
Reference in new issue
Block a user