mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
fix: avoid replaying training samples on mid-epoch resume (#2105)
This commit is contained in:
1 parent
ff2a176438
commit
c2b12c5aa5
3 files changed
+108
-4
No files matched your search
@@ -1,5 +1,9 @@
|
||||
# CHANGELOG
|
||||
|
||||
## Version 4.1.1 (2026-11-19)
|
||||
|
||||
- fix(task): avoid replaying the same training samples when resuming mid-epoch
|
||||
|
||||
## Version 4.1.0 (2026-11-17)
|
||||
|
||||
### Breaking changes
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2020 CNRS
|
||||
# Copyright (c) 2020-2025 CNRS
|
||||
# Copyright (c) 2026- pyannoteAI
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -33,12 +34,12 @@ def create_rng_for_worker(model) -> Random:
|
||||
|
||||
This makes sure that
|
||||
1. training samples generation is reproducible
|
||||
2. every (worker, epoch) uses a different seed
|
||||
2. every (worker, epoch, global step) uses a different seed
|
||||
|
||||
Parameters
|
||||
----------
|
||||
epoch : int
|
||||
Current epoch.
|
||||
model : Model
|
||||
Model providing distributed ranks, current epoch, and global step.
|
||||
"""
|
||||
|
||||
# create random number generator
|
||||
@@ -58,6 +59,7 @@ def create_rng_for_worker(model) -> Random:
|
||||
model.local_rank,
|
||||
model.global_rank,
|
||||
model.current_epoch,
|
||||
model.global_step,
|
||||
)
|
||||
# use adler32 because python's `hash` is not deterministic.
|
||||
seed = zlib.adler32(str(seed_tuple).encode())
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Sampling should be reproducible without replaying an epoch on resume."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
import lightning.pytorch as pl
|
||||
import pytest
|
||||
import torch
|
||||
from torch.utils.data import DataLoader, IterableDataset
|
||||
|
||||
from pyannote.audio.utils.random import create_rng_for_worker
|
||||
|
||||
|
||||
@pytest.mark.parametrize('worker_id', [None, 0, 1])
|
||||
def test_sampling_seed_includes_restored_progress(monkeypatch, worker_id):
|
||||
monkeypatch.setenv('PL_GLOBAL_SEED', '0')
|
||||
worker = None if worker_id is None else SimpleNamespace(id=worker_id)
|
||||
monkeypatch.setattr(torch.utils.data, 'get_worker_info', lambda: worker)
|
||||
model = SimpleNamespace(local_rank=0, global_rank=0, current_epoch=1, global_step=0)
|
||||
def samples():
|
||||
rng = create_rng_for_worker(model)
|
||||
return [rng.random() for _ in range(16)]
|
||||
opening = samples()
|
||||
model.global_step = 345000
|
||||
resumed = samples()
|
||||
assert resumed != opening
|
||||
assert samples() == resumed
|
||||
model.current_epoch += 1
|
||||
assert samples() != resumed
|
||||
|
||||
|
||||
class SamplingDataset(IterableDataset):
|
||||
def __init__(self, model):
|
||||
self.model = model
|
||||
|
||||
def __iter__(self):
|
||||
rng = create_rng_for_worker(self.model)
|
||||
while True:
|
||||
yield torch.tensor([rng.random() for _ in range(4)])
|
||||
|
||||
|
||||
class SamplingModel(pl.LightningModule):
|
||||
def __init__(self, workers):
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.ones(()))
|
||||
self.workers = workers
|
||||
self.samples = []
|
||||
self.sample_epochs = []
|
||||
|
||||
def train_dataloader(self):
|
||||
kwargs = {'multiprocessing_context': 'fork'} if self.workers else {}
|
||||
return DataLoader(SamplingDataset(self), batch_size=2,
|
||||
num_workers=self.workers, **kwargs)
|
||||
|
||||
def training_step(self, batch, batch_idx):
|
||||
self.samples.append(batch.detach().clone())
|
||||
self.sample_epochs.append(self.current_epoch)
|
||||
return self.weight * batch.mean()
|
||||
|
||||
def configure_optimizers(self):
|
||||
return torch.optim.SGD(self.parameters(), lr=.01)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('workers', [0, 2])
|
||||
def test_lightning_mid_epoch_resume_does_not_replay_opening_samples(tmp_path, monkeypatch, workers):
|
||||
if workers:
|
||||
import multiprocessing
|
||||
if 'fork' not in multiprocessing.get_all_start_methods():
|
||||
pytest.skip('pyannote training workers require fork')
|
||||
monkeypatch.setenv('PL_GLOBAL_SEED', '0')
|
||||
|
||||
checkpoint_path = tmp_path / 'mid-epoch.ckpt'
|
||||
|
||||
class SaveMidEpoch(pl.Callback):
|
||||
def on_train_batch_end(self, trainer, model, outputs, batch, batch_idx):
|
||||
if trainer.global_step == 3:
|
||||
trainer.save_checkpoint(checkpoint_path)
|
||||
|
||||
def run(steps, checkpoint=None):
|
||||
model = SamplingModel(workers)
|
||||
trainer = pl.Trainer(accelerator='cpu', devices=1, max_steps=steps,
|
||||
max_epochs=-1, limit_train_batches=1000,
|
||||
logger=False, enable_checkpointing=False,
|
||||
enable_progress_bar=False, enable_model_summary=False,
|
||||
default_root_dir=tmp_path,
|
||||
callbacks=[SaveMidEpoch()] if checkpoint is None else [])
|
||||
trainer.fit(model, ckpt_path=checkpoint)
|
||||
return model, trainer
|
||||
|
||||
initial, trainer = run(3)
|
||||
assert torch.load(checkpoint_path, weights_only=False)['epoch'] == 0
|
||||
resumed, resumed_trainer = run(5, checkpoint_path)
|
||||
repeated, _ = run(5, checkpoint_path)
|
||||
assert resumed_trainer.global_step == 5
|
||||
assert initial.sample_epochs == [0, 0, 0]
|
||||
assert resumed.sample_epochs == [0, 0]
|
||||
assert len(resumed.samples) == len(repeated.samples) == 2
|
||||
assert not torch.equal(initial.samples[0], resumed.samples[0])
|
||||
for a, b in zip(resumed.samples, repeated.samples):
|
||||
torch.testing.assert_close(a, b, rtol=0, atol=0)
|
||||
Reference in new issue
Block a user