mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
fix: avoid replaying training samples on mid-epoch resume (#2105)
This commit is contained in:
1 parent
ff2a176438
commit
c2b12c5aa5
3 files changed
+108
-4
No files matched your search
@@ -1,5 +1,9 @@
|
|||||||
# CHANGELOG
|
# CHANGELOG
|
||||||
|
|
||||||
|
## Version 4.1.1 (2026-11-19)
|
||||||
|
|
||||||
|
- fix(task): avoid replaying the same training samples when resuming mid-epoch
|
||||||
|
|
||||||
## Version 4.1.0 (2026-11-17)
|
## Version 4.1.0 (2026-11-17)
|
||||||
|
|
||||||
### Breaking changes
|
### Breaking changes
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# MIT License
|
# MIT License
|
||||||
#
|
#
|
||||||
# Copyright (c) 2020 CNRS
|
# Copyright (c) 2020-2025 CNRS
|
||||||
|
# Copyright (c) 2026- pyannoteAI
|
||||||
#
|
#
|
||||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
# of this software and associated documentation files (the "Software"), to deal
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
@@ -33,12 +34,12 @@ def create_rng_for_worker(model) -> Random:
|
|||||||
|
|
||||||
This makes sure that
|
This makes sure that
|
||||||
1. training samples generation is reproducible
|
1. training samples generation is reproducible
|
||||||
2. every (worker, epoch) uses a different seed
|
2. every (worker, epoch, global step) uses a different seed
|
||||||
|
|
||||||
Parameters
|
Parameters
|
||||||
----------
|
----------
|
||||||
epoch : int
|
model : Model
|
||||||
Current epoch.
|
Model providing distributed ranks, current epoch, and global step.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# create random number generator
|
# create random number generator
|
||||||
@@ -58,6 +59,7 @@ def create_rng_for_worker(model) -> Random:
|
|||||||
model.local_rank,
|
model.local_rank,
|
||||||
model.global_rank,
|
model.global_rank,
|
||||||
model.current_epoch,
|
model.current_epoch,
|
||||||
|
model.global_step,
|
||||||
)
|
)
|
||||||
# use adler32 because python's `hash` is not deterministic.
|
# use adler32 because python's `hash` is not deterministic.
|
||||||
seed = zlib.adler32(str(seed_tuple).encode())
|
seed = zlib.adler32(str(seed_tuple).encode())
|
||||||
|
|||||||
@@ -0,0 +1,98 @@
|
|||||||
|
"""Sampling should be reproducible without replaying an epoch on resume."""
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import lightning.pytorch as pl
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
from torch.utils.data import DataLoader, IterableDataset
|
||||||
|
|
||||||
|
from pyannote.audio.utils.random import create_rng_for_worker
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('worker_id', [None, 0, 1])
|
||||||
|
def test_sampling_seed_includes_restored_progress(monkeypatch, worker_id):
|
||||||
|
monkeypatch.setenv('PL_GLOBAL_SEED', '0')
|
||||||
|
worker = None if worker_id is None else SimpleNamespace(id=worker_id)
|
||||||
|
monkeypatch.setattr(torch.utils.data, 'get_worker_info', lambda: worker)
|
||||||
|
model = SimpleNamespace(local_rank=0, global_rank=0, current_epoch=1, global_step=0)
|
||||||
|
def samples():
|
||||||
|
rng = create_rng_for_worker(model)
|
||||||
|
return [rng.random() for _ in range(16)]
|
||||||
|
opening = samples()
|
||||||
|
model.global_step = 345000
|
||||||
|
resumed = samples()
|
||||||
|
assert resumed != opening
|
||||||
|
assert samples() == resumed
|
||||||
|
model.current_epoch += 1
|
||||||
|
assert samples() != resumed
|
||||||
|
|
||||||
|
|
||||||
|
class SamplingDataset(IterableDataset):
|
||||||
|
def __init__(self, model):
|
||||||
|
self.model = model
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
rng = create_rng_for_worker(self.model)
|
||||||
|
while True:
|
||||||
|
yield torch.tensor([rng.random() for _ in range(4)])
|
||||||
|
|
||||||
|
|
||||||
|
class SamplingModel(pl.LightningModule):
|
||||||
|
def __init__(self, workers):
|
||||||
|
super().__init__()
|
||||||
|
self.weight = torch.nn.Parameter(torch.ones(()))
|
||||||
|
self.workers = workers
|
||||||
|
self.samples = []
|
||||||
|
self.sample_epochs = []
|
||||||
|
|
||||||
|
def train_dataloader(self):
|
||||||
|
kwargs = {'multiprocessing_context': 'fork'} if self.workers else {}
|
||||||
|
return DataLoader(SamplingDataset(self), batch_size=2,
|
||||||
|
num_workers=self.workers, **kwargs)
|
||||||
|
|
||||||
|
def training_step(self, batch, batch_idx):
|
||||||
|
self.samples.append(batch.detach().clone())
|
||||||
|
self.sample_epochs.append(self.current_epoch)
|
||||||
|
return self.weight * batch.mean()
|
||||||
|
|
||||||
|
def configure_optimizers(self):
|
||||||
|
return torch.optim.SGD(self.parameters(), lr=.01)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('workers', [0, 2])
|
||||||
|
def test_lightning_mid_epoch_resume_does_not_replay_opening_samples(tmp_path, monkeypatch, workers):
|
||||||
|
if workers:
|
||||||
|
import multiprocessing
|
||||||
|
if 'fork' not in multiprocessing.get_all_start_methods():
|
||||||
|
pytest.skip('pyannote training workers require fork')
|
||||||
|
monkeypatch.setenv('PL_GLOBAL_SEED', '0')
|
||||||
|
|
||||||
|
checkpoint_path = tmp_path / 'mid-epoch.ckpt'
|
||||||
|
|
||||||
|
class SaveMidEpoch(pl.Callback):
|
||||||
|
def on_train_batch_end(self, trainer, model, outputs, batch, batch_idx):
|
||||||
|
if trainer.global_step == 3:
|
||||||
|
trainer.save_checkpoint(checkpoint_path)
|
||||||
|
|
||||||
|
def run(steps, checkpoint=None):
|
||||||
|
model = SamplingModel(workers)
|
||||||
|
trainer = pl.Trainer(accelerator='cpu', devices=1, max_steps=steps,
|
||||||
|
max_epochs=-1, limit_train_batches=1000,
|
||||||
|
logger=False, enable_checkpointing=False,
|
||||||
|
enable_progress_bar=False, enable_model_summary=False,
|
||||||
|
default_root_dir=tmp_path,
|
||||||
|
callbacks=[SaveMidEpoch()] if checkpoint is None else [])
|
||||||
|
trainer.fit(model, ckpt_path=checkpoint)
|
||||||
|
return model, trainer
|
||||||
|
|
||||||
|
initial, trainer = run(3)
|
||||||
|
assert torch.load(checkpoint_path, weights_only=False)['epoch'] == 0
|
||||||
|
resumed, resumed_trainer = run(5, checkpoint_path)
|
||||||
|
repeated, _ = run(5, checkpoint_path)
|
||||||
|
assert resumed_trainer.global_step == 5
|
||||||
|
assert initial.sample_epochs == [0, 0, 0]
|
||||||
|
assert resumed.sample_epochs == [0, 0]
|
||||||
|
assert len(resumed.samples) == len(repeated.samples) == 2
|
||||||
|
assert not torch.equal(initial.samples[0], resumed.samples[0])
|
||||||
|
for a, b in zip(resumed.samples, repeated.samples):
|
||||||
|
torch.testing.assert_close(a, b, rtol=0, atol=0)
|
||||||
Reference in new issue
Block a user