feat: add pyannote-audio-eval tool (#888)

This commit is contained in:
Hervé BREDIN authored and GitHub committed 2022-02-14 13:54:21 +01:00
1 parent 6845ace2da
commit f187fa423f
26 files changed
+185 -4

No files matched your search

@@ -1,2 +0,0 @@
# @package _group_
_target_: pyannote.audio.models.segmentation.PyanNet
+80
View File
@@ -0,0 +1,80 @@
# MIT License
#
# Copyright (c) 2022- CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import Optional
from pyannote.audio import Inference, Model
from pyannote.audio.pipelines.utils import get_devices
from pyannote.audio.utils.signal import binarize
from pyannote.audio.utils.metric import DiscreteDiarizationErrorRate
from pyannote.database import get_protocol, FileFinder
from rich.progress import Progress
import hydra
from omegaconf import DictConfig
from pyannote.database import FileFinder, get_protocol, ProtocolFile
@hydra.main(config_path="evaluate_config", config_name="config")
def evaluate(cfg: DictConfig) -> Optional[float]:
# load pretrained model
(device,) = get_devices(needs=1)
model = Model.from_pretrained(cfg.model, device=device)
# load evaluation files
protocol = get_protocol(cfg.protocol, preprocessors={"audio": FileFinder()})
files = list(getattr(protocol, cfg.subset)())
# load evaluation metric
metric = DiscreteDiarizationErrorRate()
with Progress() as progress:
main_task = progress.add_task(protocol.name, total=len(files))
file_task = progress.add_task("Processing", total=1.0)
def progress_hook(completed: int, total: int):
progress.update(file_task, completed=completed / total)
inference = Inference(model, device=device, progress_hook=progress_hook)
warm_up = cfg.warm_up / inference.duration
def hypothesis(file: ProtocolFile):
return Inference.trim(binarize(inference(file)), warm_up=(warm_up, warm_up))
metric = DiscreteDiarizationErrorRate()
for file in files:
progress.update(file_task, description=file["uri"])
reference = file["annotation"]
uem = file["annotated"]
_ = metric(reference, hypothesis(file), uem=uem)
progress.advance(main_task)
_ = metric.report(display=True)
if __name__ == "__main__":
evaluate()
@@ -0,0 +1,7 @@
model: ???
protocol: ???
warm_up: 0.0
subset: test
defaults:
- hydra: default
@@ -0,0 +1,30 @@
# @package _group_
run:
dir: ${protocol}/${now:%Y-%m-%dT%H:%M:%S.%fZ}
sweep:
dir: ${protocol}/${now:%Y-%m-%dT%H:%M:%S.%fZ}
subdir: ${hydra.job.num}
output_subdir: ""
help:
app_name: pyannote-audio-eval
# Help header, customize to describe your app to your users
header: == ${hydra.help.app_name} ==
footer: |-
Powered by Hydra (https://hydra.cc)
Use --hydra-help to view Hydra specific help
template: |-
${hydra.help.header}
pyannote-audio-eval protocol={protocol_name}
subset={test | development | train}
model={path_to_pretrained_model}
warm_up={warm_up_duration_in_seconds}
${hydra.help.footer}
+2 -2
View File
@@ -1,6 +1,6 @@
# MIT License
#
# Copyright (c) 2020-2021 CNRS
# Copyright (c) 2020-2022 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -42,7 +42,7 @@ from torch_audiomentations.utils.config import from_dict as get_augmentation
from pyannote.database import FileFinder, get_protocol
@hydra.main(config_path="config", config_name="config")
@hydra.main(config_path="train_config", config_name="config")
def train(cfg: DictConfig) -> Optional[float]:
# make sure to set the random seed before the instantiation of Trainer
@@ -0,0 +1,21 @@
# MIT License
#
# Copyright (c) 2020-2021 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
@@ -0,0 +1,31 @@
# @package _group_
run:
dir: ${protocol}/${now:%Y-%m-%dT%H:%M:%S.%fZ}
sweep:
dir: ${protocol}/${now:%Y-%m-%dT%H:%M:%S.%fZ}
subdir: ${hydra.job.num}
output_subdir: ""
help:
app_name: pyannote-audio-train
# Help header, customize to describe your app to your users
header: == ${hydra.help.app_name} ==
footer: |-
Powered by Hydra (https://hydra.cc)
Use --hydra-help to view Hydra specific help
template: |-
${hydra.help.header}
pyannote-audio-train protocol={protocol_name}
+task={task} task.param=...
+model={model} model.param=...
optimizer={optimizer} optimizer.param=...
scheduler={scheduler} scheduler.param=...
${hydra.help.footer}
@@ -0,0 +1,13 @@
# @package _group_
_target_: pyannote.audio.models.segmentation.PyanNet
sincnet:
stride: 10
lstm:
hidden_size: 128
num_layers: 2
bidirectional: true
monolithic: true
dropout: 0.5
linear:
hidden_size: 128
num_layers: 2
+1
View File
@@ -59,6 +59,7 @@ cli =
console_scripts =
pyannote-audio-train=pyannote.audio.cli.train:train
pyannote-audio-eval=pyannote.audio.cli.evaluate:evaluate
prodigy_recipes =
audio.vad = pyannote.audio.interactive.recipes.voice_activity_detection:voice_activity_detection