fix(cli): update CLI with latest APIs

This commit is contained in:
Clément Pagés authored and GitHub committed 2024-02-25 20:49:08 +01:00
1 parent 5b3ed06129
commit d83c0f8ea9
8 files changed
+205 -15

No files matched your search

+1 -1
View File
@@ -30,4 +30,4 @@ jobs:
pip install -e .[dev,testing]
- name: Test with pytest
run: |
pytest
pytest -k "not test_cli.py"
+33
View File
@@ -0,0 +1,33 @@
name: CLI tests
on:
push:
branches: [develop]
pull_request:
branches: [develop]
jobs:
build:
timeout-minutes: 20
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest]
python-version: ["3.10"]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install libsndfile
if: matrix.os == 'ubuntu-latest'
run: |
sudo apt-get update
sudo apt-get install libsndfile1
- name: Install pyannote.audio
run: |
pip install -e .[dev,testing,cli]
- name: Test with pytest
run: |
pytest tests/test_cli.py
+10 -4
View File
@@ -25,7 +25,7 @@ from typing import Optional
import hydra
from omegaconf import DictConfig
from pyannote.database import FileFinder, ProtocolFile, get_protocol
from pyannote.database import FileFinder, ProtocolFile, registry
from rich.progress import Progress
from pyannote.audio import Inference, Model
@@ -41,8 +41,16 @@ def evaluate(cfg: DictConfig) -> Optional[float]:
(device,) = get_devices(needs=1)
model = Model.from_pretrained(cfg.model, device=device)
# load databases into registry if it was specified
if "registry" in cfg:
for database_yml in cfg.registry.split(","):
registry.load_database(database_yml)
# load evaluation files
protocol = get_protocol(cfg.protocol, preprocessors={"audio": FileFinder()})
protocol = registry.get_protocol(
cfg.protocol, preprocessors={"audio": FileFinder()}
)
files = list(getattr(protocol, cfg.subset)())
# load evaluation metric
@@ -65,8 +73,6 @@ def evaluate(cfg: DictConfig) -> Optional[float]:
warm_up=(warm_up, warm_up),
)
metric = DiscreteDiarizationErrorRate()
for file in files:
progress.update(file_task, description=file["uri"])
reference = file["annotation"]
@@ -22,7 +22,8 @@ help:
template: |-
${hydra.help.header}
pyannote-audio-eval protocol={protocol_name}
pyannote-audio-eval registry={path_to_database.yml}
protocol={protocol_name}
subset={test | development | train}
model={path_to_pretrained_model}
warm_up={warm_up_duration_in_seconds}
+9 -7
View File
@@ -51,8 +51,9 @@ def train(cfg: DictConfig) -> Optional[float]:
seed_everything(seed=seed)
# load databases into registry
for database_yml in cfg.registry.split(","):
registry.load_database(database_yml)
if "registry" in cfg:
for database_yml in cfg.registry.split(","):
registry.load_database(database_yml)
# instantiate training protocol with optional preprocessors
preprocessors = {"audio": FileFinder(), "torchaudio.info": get_torchaudio_info}
@@ -75,9 +76,9 @@ def train(cfg: DictConfig) -> Optional[float]:
# instantiate model
fine_tuning = cfg.model["_target_"] == "pyannote.audio.cli.pretrained"
model = instantiate(cfg.model)
model.task = task
model.setup(stage="fit")
model = instantiate(cfg.model, task=task)
model.prepare_data()
model.setup()
# validation metric to monitor (and its direction: min or max)
monitor, direction = task.val_monitor
@@ -146,7 +147,6 @@ def train(cfg: DictConfig) -> Optional[float]:
# in case of fine-tuning, validate the initial model to make sure
# that we actually improve over the initial performance
if fine_tuning:
model.setup(stage="fit")
trainer.validate(model)
# train the model
@@ -157,7 +157,9 @@ def train(cfg: DictConfig) -> Optional[float]:
# return the best validation score
# this can be used for hyper-parameter optimization with Hydra sweepers
if monitor is not None:
# this can only be done if the trainer is not in fast dev run mode, as
# checkpointing is disabled in this mode
if monitor is not None and not trainer.fast_dev_run:
best_monitor = float(checkpoint.best_model_score)
if direction == "min":
return best_monitor
@@ -1,4 +1,3 @@
registry: ???
protocol: ???
defaults:
@@ -30,6 +30,6 @@ precision: 32
profiler: null
reload_dataloaders_every_n_epochs: 0
use_distributed_sampler: True # TODO: check what this does exactly
strategy: null
strategy: auto
sync_batchnorm: False
val_check_interval: 1.0
+149
View File
@@ -0,0 +1,149 @@
# The MIT License (MIT)
#
# Copyright (c) 2024- CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import subprocess
import pytest
from pyannote.database import FileFinder, get_protocol
@pytest.fixture()
def protocol():
return get_protocol(
"Debug.SpeakerDiarization.Debug", preprocessors={"audio": FileFinder()}
)
@pytest.fixture()
def database():
return "./tests/data/database.yml"
@pytest.fixture()
def model():
return "pyannote/ci-segmentation"
def test_cli_train_vad(database, protocol):
res = subprocess.run(
[
"pyannote-audio-train",
"model=DebugSegmentation",
"task=VoiceActivityDetection",
f"+registry={database}",
f"protocol={protocol.name}",
"trainer=fast_dev_run",
"hydra.run.dir=.", # run hydra app in current directory
"hydra.output_subdir=null", # disable hydra outputs
"hydra/hydra_logging=disabled",
"hydra/job_logging=disabled",
]
)
assert res.returncode == 0
def test_cli_train_segmentation(database, protocol):
res = subprocess.run(
[
"pyannote-audio-train",
"model=DebugSegmentation",
"task=SpeakerDiarization",
f"+registry={database}",
f"protocol={protocol.name}",
"trainer=fast_dev_run",
"hydra.run.dir=.", # run hydra app in current directory
"hydra.output_subdir=null", # disable hydra outputs
"hydra/hydra_logging=disabled",
"hydra/job_logging=disabled",
]
)
assert res.returncode == 0
def test_cli_train_osd(database, protocol):
res = subprocess.run(
[
"pyannote-audio-train",
"model=DebugSegmentation",
"task=OverlappedSpeechDetection",
f"+registry={database}",
f"protocol={protocol.name}",
"trainer=fast_dev_run",
"hydra.run.dir=.", # run hydra app in current directory
"hydra.output_subdir=null", # disable hydra outputs
"hydra/hydra_logging=disabled",
"hydra/job_logging=disabled",
]
)
assert res.returncode == 0
def test_cli_train_supervised_representation_with_arcface(database, protocol):
res = subprocess.run(
[
"pyannote-audio-train",
"model=DebugEmbedding",
"task=SpeakerEmbedding",
f"+registry={database}",
f"protocol={protocol.name}",
"trainer=fast_dev_run",
"hydra.run.dir=.", # run hydra app in current directory
"hydra.output_subdir=null", # disable hydra outputs
"hydra/hydra_logging=disabled",
"hydra/job_logging=disabled",
]
)
assert res.returncode == 0
def test_cli_train_segmentation_with_pyannet(database, protocol):
res = subprocess.run(
[
"pyannote-audio-train",
"model=PyanNet",
"task=SpeakerDiarization",
f"+registry={database}",
f"protocol={protocol.name}",
"trainer=fast_dev_run",
"hydra.run.dir=.", # run hydra app in current directory
"hydra.output_subdir=null", # disable hydra outputs
"hydra/hydra_logging=disabled",
"hydra/job_logging=disabled",
]
)
assert res.returncode == 0
def test_cli_eval_segmentation_model(database, protocol, model):
res = subprocess.run(
[
"pyannote-audio-eval",
f"model={model}",
f"+registry={database}",
f"protocol={protocol.name}",
"hydra.run.dir=.", # run hydra app in current directory
"hydra.output_subdir=null", # disable hydra outputs
"hydra/hydra_logging=disabled",
"hydra/job_logging=disabled",
]
)
assert res.returncode == 0