mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
fix(cli): update CLI with latest APIs
This commit is contained in:
1 parent
5b3ed06129
commit
d83c0f8ea9
8 files changed
+205
-15
No files matched your search
@@ -30,4 +30,4 @@ jobs:
|
||||
pip install -e .[dev,testing]
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
pytest
|
||||
pytest -k "not test_cli.py"
|
||||
@@ -0,0 +1,33 @@
|
||||
name: CLI tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [develop]
|
||||
pull_request:
|
||||
branches: [develop]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
timeout-minutes: 20
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
python-version: ["3.10"]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install libsndfile
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install libsndfile1
|
||||
- name: Install pyannote.audio
|
||||
run: |
|
||||
pip install -e .[dev,testing,cli]
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
pytest tests/test_cli.py
|
||||
@@ -25,7 +25,7 @@ from typing import Optional
|
||||
|
||||
import hydra
|
||||
from omegaconf import DictConfig
|
||||
from pyannote.database import FileFinder, ProtocolFile, get_protocol
|
||||
from pyannote.database import FileFinder, ProtocolFile, registry
|
||||
from rich.progress import Progress
|
||||
|
||||
from pyannote.audio import Inference, Model
|
||||
@@ -41,8 +41,16 @@ def evaluate(cfg: DictConfig) -> Optional[float]:
|
||||
(device,) = get_devices(needs=1)
|
||||
model = Model.from_pretrained(cfg.model, device=device)
|
||||
|
||||
# load databases into registry if it was specified
|
||||
if "registry" in cfg:
|
||||
for database_yml in cfg.registry.split(","):
|
||||
registry.load_database(database_yml)
|
||||
|
||||
# load evaluation files
|
||||
protocol = get_protocol(cfg.protocol, preprocessors={"audio": FileFinder()})
|
||||
protocol = registry.get_protocol(
|
||||
cfg.protocol, preprocessors={"audio": FileFinder()}
|
||||
)
|
||||
|
||||
files = list(getattr(protocol, cfg.subset)())
|
||||
|
||||
# load evaluation metric
|
||||
@@ -65,8 +73,6 @@ def evaluate(cfg: DictConfig) -> Optional[float]:
|
||||
warm_up=(warm_up, warm_up),
|
||||
)
|
||||
|
||||
metric = DiscreteDiarizationErrorRate()
|
||||
|
||||
for file in files:
|
||||
progress.update(file_task, description=file["uri"])
|
||||
reference = file["annotation"]
|
||||
|
||||
@@ -22,7 +22,8 @@ help:
|
||||
template: |-
|
||||
${hydra.help.header}
|
||||
|
||||
pyannote-audio-eval protocol={protocol_name}
|
||||
pyannote-audio-eval registry={path_to_database.yml}
|
||||
protocol={protocol_name}
|
||||
subset={test | development | train}
|
||||
model={path_to_pretrained_model}
|
||||
warm_up={warm_up_duration_in_seconds}
|
||||
|
||||
@@ -51,8 +51,9 @@ def train(cfg: DictConfig) -> Optional[float]:
|
||||
seed_everything(seed=seed)
|
||||
|
||||
# load databases into registry
|
||||
for database_yml in cfg.registry.split(","):
|
||||
registry.load_database(database_yml)
|
||||
if "registry" in cfg:
|
||||
for database_yml in cfg.registry.split(","):
|
||||
registry.load_database(database_yml)
|
||||
|
||||
# instantiate training protocol with optional preprocessors
|
||||
preprocessors = {"audio": FileFinder(), "torchaudio.info": get_torchaudio_info}
|
||||
@@ -75,9 +76,9 @@ def train(cfg: DictConfig) -> Optional[float]:
|
||||
|
||||
# instantiate model
|
||||
fine_tuning = cfg.model["_target_"] == "pyannote.audio.cli.pretrained"
|
||||
model = instantiate(cfg.model)
|
||||
model.task = task
|
||||
model.setup(stage="fit")
|
||||
model = instantiate(cfg.model, task=task)
|
||||
model.prepare_data()
|
||||
model.setup()
|
||||
|
||||
# validation metric to monitor (and its direction: min or max)
|
||||
monitor, direction = task.val_monitor
|
||||
@@ -146,7 +147,6 @@ def train(cfg: DictConfig) -> Optional[float]:
|
||||
# in case of fine-tuning, validate the initial model to make sure
|
||||
# that we actually improve over the initial performance
|
||||
if fine_tuning:
|
||||
model.setup(stage="fit")
|
||||
trainer.validate(model)
|
||||
|
||||
# train the model
|
||||
@@ -157,7 +157,9 @@ def train(cfg: DictConfig) -> Optional[float]:
|
||||
|
||||
# return the best validation score
|
||||
# this can be used for hyper-parameter optimization with Hydra sweepers
|
||||
if monitor is not None:
|
||||
# this can only be done if the trainer is not in fast dev run mode, as
|
||||
# checkpointing is disabled in this mode
|
||||
if monitor is not None and not trainer.fast_dev_run:
|
||||
best_monitor = float(checkpoint.best_model_score)
|
||||
if direction == "min":
|
||||
return best_monitor
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
registry: ???
|
||||
protocol: ???
|
||||
|
||||
defaults:
|
||||
|
||||
@@ -30,6 +30,6 @@ precision: 32
|
||||
profiler: null
|
||||
reload_dataloaders_every_n_epochs: 0
|
||||
use_distributed_sampler: True # TODO: check what this does exactly
|
||||
strategy: null
|
||||
strategy: auto
|
||||
sync_batchnorm: False
|
||||
val_check_interval: 1.0
|
||||
@@ -0,0 +1,149 @@
|
||||
# The MIT License (MIT)
|
||||
#
|
||||
# Copyright (c) 2024- CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
from pyannote.database import FileFinder, get_protocol
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def protocol():
|
||||
return get_protocol(
|
||||
"Debug.SpeakerDiarization.Debug", preprocessors={"audio": FileFinder()}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def database():
|
||||
return "./tests/data/database.yml"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def model():
|
||||
return "pyannote/ci-segmentation"
|
||||
|
||||
|
||||
def test_cli_train_vad(database, protocol):
|
||||
res = subprocess.run(
|
||||
[
|
||||
"pyannote-audio-train",
|
||||
"model=DebugSegmentation",
|
||||
"task=VoiceActivityDetection",
|
||||
f"+registry={database}",
|
||||
f"protocol={protocol.name}",
|
||||
"trainer=fast_dev_run",
|
||||
"hydra.run.dir=.", # run hydra app in current directory
|
||||
"hydra.output_subdir=null", # disable hydra outputs
|
||||
"hydra/hydra_logging=disabled",
|
||||
"hydra/job_logging=disabled",
|
||||
]
|
||||
)
|
||||
assert res.returncode == 0
|
||||
|
||||
|
||||
def test_cli_train_segmentation(database, protocol):
|
||||
res = subprocess.run(
|
||||
[
|
||||
"pyannote-audio-train",
|
||||
"model=DebugSegmentation",
|
||||
"task=SpeakerDiarization",
|
||||
f"+registry={database}",
|
||||
f"protocol={protocol.name}",
|
||||
"trainer=fast_dev_run",
|
||||
"hydra.run.dir=.", # run hydra app in current directory
|
||||
"hydra.output_subdir=null", # disable hydra outputs
|
||||
"hydra/hydra_logging=disabled",
|
||||
"hydra/job_logging=disabled",
|
||||
]
|
||||
)
|
||||
assert res.returncode == 0
|
||||
|
||||
|
||||
def test_cli_train_osd(database, protocol):
|
||||
res = subprocess.run(
|
||||
[
|
||||
"pyannote-audio-train",
|
||||
"model=DebugSegmentation",
|
||||
"task=OverlappedSpeechDetection",
|
||||
f"+registry={database}",
|
||||
f"protocol={protocol.name}",
|
||||
"trainer=fast_dev_run",
|
||||
"hydra.run.dir=.", # run hydra app in current directory
|
||||
"hydra.output_subdir=null", # disable hydra outputs
|
||||
"hydra/hydra_logging=disabled",
|
||||
"hydra/job_logging=disabled",
|
||||
]
|
||||
)
|
||||
assert res.returncode == 0
|
||||
|
||||
|
||||
def test_cli_train_supervised_representation_with_arcface(database, protocol):
|
||||
res = subprocess.run(
|
||||
[
|
||||
"pyannote-audio-train",
|
||||
"model=DebugEmbedding",
|
||||
"task=SpeakerEmbedding",
|
||||
f"+registry={database}",
|
||||
f"protocol={protocol.name}",
|
||||
"trainer=fast_dev_run",
|
||||
"hydra.run.dir=.", # run hydra app in current directory
|
||||
"hydra.output_subdir=null", # disable hydra outputs
|
||||
"hydra/hydra_logging=disabled",
|
||||
"hydra/job_logging=disabled",
|
||||
]
|
||||
)
|
||||
assert res.returncode == 0
|
||||
|
||||
|
||||
def test_cli_train_segmentation_with_pyannet(database, protocol):
|
||||
res = subprocess.run(
|
||||
[
|
||||
"pyannote-audio-train",
|
||||
"model=PyanNet",
|
||||
"task=SpeakerDiarization",
|
||||
f"+registry={database}",
|
||||
f"protocol={protocol.name}",
|
||||
"trainer=fast_dev_run",
|
||||
"hydra.run.dir=.", # run hydra app in current directory
|
||||
"hydra.output_subdir=null", # disable hydra outputs
|
||||
"hydra/hydra_logging=disabled",
|
||||
"hydra/job_logging=disabled",
|
||||
]
|
||||
)
|
||||
assert res.returncode == 0
|
||||
|
||||
|
||||
def test_cli_eval_segmentation_model(database, protocol, model):
|
||||
res = subprocess.run(
|
||||
[
|
||||
"pyannote-audio-eval",
|
||||
f"model={model}",
|
||||
f"+registry={database}",
|
||||
f"protocol={protocol.name}",
|
||||
"hydra.run.dir=.", # run hydra app in current directory
|
||||
"hydra.output_subdir=null", # disable hydra outputs
|
||||
"hydra/hydra_logging=disabled",
|
||||
"hydra/job_logging=disabled",
|
||||
]
|
||||
)
|
||||
assert res.returncode == 0
|
||||
Reference in new issue
Block a user