diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90a4302c..e649027a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,4 +30,4 @@ jobs: pip install -e .[dev,testing] - name: Test with pytest run: | - pytest + pytest -k "not test_cli.py" diff --git a/.github/workflows/test_cli.yml b/.github/workflows/test_cli.yml new file mode 100644 index 00000000..731a4b5b --- /dev/null +++ b/.github/workflows/test_cli.yml @@ -0,0 +1,33 @@ +name: CLI tests + +on: + push: + branches: [develop] + pull_request: + branches: [develop] + +jobs: + build: + timeout-minutes: 20 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.10"] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install libsndfile + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get update + sudo apt-get install libsndfile1 + - name: Install pyannote.audio + run: | + pip install -e .[dev,testing,cli] + - name: Test with pytest + run: | + pytest tests/test_cli.py diff --git a/pyannote/audio/cli/evaluate.py b/pyannote/audio/cli/evaluate.py index d88c8863..3eec5e3c 100644 --- a/pyannote/audio/cli/evaluate.py +++ b/pyannote/audio/cli/evaluate.py @@ -25,7 +25,7 @@ from typing import Optional import hydra from omegaconf import DictConfig -from pyannote.database import FileFinder, ProtocolFile, get_protocol +from pyannote.database import FileFinder, ProtocolFile, registry from rich.progress import Progress from pyannote.audio import Inference, Model @@ -41,8 +41,16 @@ def evaluate(cfg: DictConfig) -> Optional[float]: (device,) = get_devices(needs=1) model = Model.from_pretrained(cfg.model, device=device) + # load databases into registry if it was specified + if "registry" in cfg: + for database_yml in cfg.registry.split(","): + registry.load_database(database_yml) + # load evaluation files - protocol = get_protocol(cfg.protocol, preprocessors={"audio": FileFinder()}) + protocol = registry.get_protocol( + cfg.protocol, preprocessors={"audio": FileFinder()} + ) + files = list(getattr(protocol, cfg.subset)()) # load evaluation metric @@ -65,8 +73,6 @@ def evaluate(cfg: DictConfig) -> Optional[float]: warm_up=(warm_up, warm_up), ) - metric = DiscreteDiarizationErrorRate() - for file in files: progress.update(file_task, description=file["uri"]) reference = file["annotation"] diff --git a/pyannote/audio/cli/evaluate_config/hydra/default.yaml b/pyannote/audio/cli/evaluate_config/hydra/default.yaml index 95979432..1aaebd1b 100644 --- a/pyannote/audio/cli/evaluate_config/hydra/default.yaml +++ b/pyannote/audio/cli/evaluate_config/hydra/default.yaml @@ -22,7 +22,8 @@ help: template: |- ${hydra.help.header} - pyannote-audio-eval protocol={protocol_name} + pyannote-audio-eval registry={path_to_database.yml} + protocol={protocol_name} subset={test | development | train} model={path_to_pretrained_model} warm_up={warm_up_duration_in_seconds} diff --git a/pyannote/audio/cli/train.py b/pyannote/audio/cli/train.py index 74041554..f052cd5b 100644 --- a/pyannote/audio/cli/train.py +++ b/pyannote/audio/cli/train.py @@ -51,8 +51,9 @@ def train(cfg: DictConfig) -> Optional[float]: seed_everything(seed=seed) # load databases into registry - for database_yml in cfg.registry.split(","): - registry.load_database(database_yml) + if "registry" in cfg: + for database_yml in cfg.registry.split(","): + registry.load_database(database_yml) # instantiate training protocol with optional preprocessors preprocessors = {"audio": FileFinder(), "torchaudio.info": get_torchaudio_info} @@ -75,9 +76,9 @@ def train(cfg: DictConfig) -> Optional[float]: # instantiate model fine_tuning = cfg.model["_target_"] == "pyannote.audio.cli.pretrained" - model = instantiate(cfg.model) - model.task = task - model.setup(stage="fit") + model = instantiate(cfg.model, task=task) + model.prepare_data() + model.setup() # validation metric to monitor (and its direction: min or max) monitor, direction = task.val_monitor @@ -146,7 +147,6 @@ def train(cfg: DictConfig) -> Optional[float]: # in case of fine-tuning, validate the initial model to make sure # that we actually improve over the initial performance if fine_tuning: - model.setup(stage="fit") trainer.validate(model) # train the model @@ -157,7 +157,9 @@ def train(cfg: DictConfig) -> Optional[float]: # return the best validation score # this can be used for hyper-parameter optimization with Hydra sweepers - if monitor is not None: + # this can only be done if the trainer is not in fast dev run mode, as + # checkpointing is disabled in this mode + if monitor is not None and not trainer.fast_dev_run: best_monitor = float(checkpoint.best_model_score) if direction == "min": return best_monitor diff --git a/pyannote/audio/cli/train_config/config.yaml b/pyannote/audio/cli/train_config/config.yaml index d5b761cc..d7ecd547 100644 --- a/pyannote/audio/cli/train_config/config.yaml +++ b/pyannote/audio/cli/train_config/config.yaml @@ -1,4 +1,3 @@ -registry: ??? protocol: ??? defaults: diff --git a/pyannote/audio/cli/train_config/trainer/default.yaml b/pyannote/audio/cli/train_config/trainer/default.yaml index ac3a60ff..f32031af 100644 --- a/pyannote/audio/cli/train_config/trainer/default.yaml +++ b/pyannote/audio/cli/train_config/trainer/default.yaml @@ -30,6 +30,6 @@ precision: 32 profiler: null reload_dataloaders_every_n_epochs: 0 use_distributed_sampler: True # TODO: check what this does exactly -strategy: null +strategy: auto sync_batchnorm: False val_check_interval: 1.0 diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..f62cc526 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,149 @@ +# The MIT License (MIT) +# +# Copyright (c) 2024- CNRS +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import subprocess + +import pytest +from pyannote.database import FileFinder, get_protocol + + +@pytest.fixture() +def protocol(): + return get_protocol( + "Debug.SpeakerDiarization.Debug", preprocessors={"audio": FileFinder()} + ) + + +@pytest.fixture() +def database(): + return "./tests/data/database.yml" + + +@pytest.fixture() +def model(): + return "pyannote/ci-segmentation" + + +def test_cli_train_vad(database, protocol): + res = subprocess.run( + [ + "pyannote-audio-train", + "model=DebugSegmentation", + "task=VoiceActivityDetection", + f"+registry={database}", + f"protocol={protocol.name}", + "trainer=fast_dev_run", + "hydra.run.dir=.", # run hydra app in current directory + "hydra.output_subdir=null", # disable hydra outputs + "hydra/hydra_logging=disabled", + "hydra/job_logging=disabled", + ] + ) + assert res.returncode == 0 + + +def test_cli_train_segmentation(database, protocol): + res = subprocess.run( + [ + "pyannote-audio-train", + "model=DebugSegmentation", + "task=SpeakerDiarization", + f"+registry={database}", + f"protocol={protocol.name}", + "trainer=fast_dev_run", + "hydra.run.dir=.", # run hydra app in current directory + "hydra.output_subdir=null", # disable hydra outputs + "hydra/hydra_logging=disabled", + "hydra/job_logging=disabled", + ] + ) + assert res.returncode == 0 + + +def test_cli_train_osd(database, protocol): + res = subprocess.run( + [ + "pyannote-audio-train", + "model=DebugSegmentation", + "task=OverlappedSpeechDetection", + f"+registry={database}", + f"protocol={protocol.name}", + "trainer=fast_dev_run", + "hydra.run.dir=.", # run hydra app in current directory + "hydra.output_subdir=null", # disable hydra outputs + "hydra/hydra_logging=disabled", + "hydra/job_logging=disabled", + ] + ) + assert res.returncode == 0 + + +def test_cli_train_supervised_representation_with_arcface(database, protocol): + res = subprocess.run( + [ + "pyannote-audio-train", + "model=DebugEmbedding", + "task=SpeakerEmbedding", + f"+registry={database}", + f"protocol={protocol.name}", + "trainer=fast_dev_run", + "hydra.run.dir=.", # run hydra app in current directory + "hydra.output_subdir=null", # disable hydra outputs + "hydra/hydra_logging=disabled", + "hydra/job_logging=disabled", + ] + ) + assert res.returncode == 0 + + +def test_cli_train_segmentation_with_pyannet(database, protocol): + res = subprocess.run( + [ + "pyannote-audio-train", + "model=PyanNet", + "task=SpeakerDiarization", + f"+registry={database}", + f"protocol={protocol.name}", + "trainer=fast_dev_run", + "hydra.run.dir=.", # run hydra app in current directory + "hydra.output_subdir=null", # disable hydra outputs + "hydra/hydra_logging=disabled", + "hydra/job_logging=disabled", + ] + ) + assert res.returncode == 0 + + +def test_cli_eval_segmentation_model(database, protocol, model): + res = subprocess.run( + [ + "pyannote-audio-eval", + f"model={model}", + f"+registry={database}", + f"protocol={protocol.name}", + "hydra.run.dir=.", # run hydra app in current directory + "hydra.output_subdir=null", # disable hydra outputs + "hydra/hydra_logging=disabled", + "hydra/job_logging=disabled", + ] + ) + assert res.returncode == 0