From 88b14f6ce143847fff82faa26ca3be2b08eb74e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20BREDIN?= Date: Wed, 14 Jun 2023 22:33:41 +0200 Subject: [PATCH] feat: use SWA when finetuning --- pyannote/audio/cli/train.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pyannote/audio/cli/train.py b/pyannote/audio/cli/train.py index 9ab8b165..b3b380b6 100644 --- a/pyannote/audio/cli/train.py +++ b/pyannote/audio/cli/train.py @@ -36,6 +36,7 @@ from pytorch_lightning.callbacks import ( LearningRateMonitor, ModelCheckpoint, RichProgressBar, + StochasticWeightAveraging, ) from pytorch_lightning.loggers import TensorBoardLogger from torch_audiomentations.utils.config import from_dict as get_augmentation @@ -110,7 +111,12 @@ def train(cfg: DictConfig) -> Optional[float]: # TODO: for fine-tuning and/or transfer learning, we start by fitting # TODO: task-dependent layers and gradully unfreeze more layers # TODO: callbacks.append(GraduallyUnfreeze(epochs_per_stage=1)) - pass + swa = StochasticWeightAveraging( + 1e-2, + swa_epoch_start=2, + annealing_epochs=10, + ) + callbacks.append(swa) checkpoint = ModelCheckpoint( monitor=monitor,