mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
feat: add pyannote-audio-eval tool (#888)
This commit is contained in:
1 parent
6845ace2da
commit
f187fa423f
26 files changed
+185
-4
No files matched your search
@@ -1,2 +0,0 @@
|
||||
# @package _group_
|
||||
_target_: pyannote.audio.models.segmentation.PyanNet
|
||||
@@ -0,0 +1,80 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2022- CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
|
||||
from typing import Optional
|
||||
from pyannote.audio import Inference, Model
|
||||
from pyannote.audio.pipelines.utils import get_devices
|
||||
from pyannote.audio.utils.signal import binarize
|
||||
from pyannote.audio.utils.metric import DiscreteDiarizationErrorRate
|
||||
from pyannote.database import get_protocol, FileFinder
|
||||
from rich.progress import Progress
|
||||
|
||||
import hydra
|
||||
from omegaconf import DictConfig
|
||||
|
||||
from pyannote.database import FileFinder, get_protocol, ProtocolFile
|
||||
|
||||
|
||||
@hydra.main(config_path="evaluate_config", config_name="config")
|
||||
def evaluate(cfg: DictConfig) -> Optional[float]:
|
||||
|
||||
# load pretrained model
|
||||
(device,) = get_devices(needs=1)
|
||||
model = Model.from_pretrained(cfg.model, device=device)
|
||||
|
||||
# load evaluation files
|
||||
protocol = get_protocol(cfg.protocol, preprocessors={"audio": FileFinder()})
|
||||
files = list(getattr(protocol, cfg.subset)())
|
||||
|
||||
# load evaluation metric
|
||||
metric = DiscreteDiarizationErrorRate()
|
||||
|
||||
with Progress() as progress:
|
||||
|
||||
main_task = progress.add_task(protocol.name, total=len(files))
|
||||
file_task = progress.add_task("Processing", total=1.0)
|
||||
|
||||
def progress_hook(completed: int, total: int):
|
||||
progress.update(file_task, completed=completed / total)
|
||||
|
||||
inference = Inference(model, device=device, progress_hook=progress_hook)
|
||||
warm_up = cfg.warm_up / inference.duration
|
||||
|
||||
def hypothesis(file: ProtocolFile):
|
||||
return Inference.trim(binarize(inference(file)), warm_up=(warm_up, warm_up))
|
||||
|
||||
metric = DiscreteDiarizationErrorRate()
|
||||
|
||||
for file in files:
|
||||
progress.update(file_task, description=file["uri"])
|
||||
reference = file["annotation"]
|
||||
uem = file["annotated"]
|
||||
_ = metric(reference, hypothesis(file), uem=uem)
|
||||
progress.advance(main_task)
|
||||
|
||||
_ = metric.report(display=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
evaluate()
|
||||
|
||||
File renamed without changes.
@@ -0,0 +1,7 @@
|
||||
model: ???
|
||||
protocol: ???
|
||||
warm_up: 0.0
|
||||
subset: test
|
||||
|
||||
defaults:
|
||||
- hydra: default
|
||||
@@ -0,0 +1,30 @@
|
||||
# @package _group_
|
||||
|
||||
run:
|
||||
dir: ${protocol}/${now:%Y-%m-%dT%H:%M:%S.%fZ}
|
||||
|
||||
sweep:
|
||||
dir: ${protocol}/${now:%Y-%m-%dT%H:%M:%S.%fZ}
|
||||
subdir: ${hydra.job.num}
|
||||
|
||||
output_subdir: ""
|
||||
|
||||
help:
|
||||
app_name: pyannote-audio-eval
|
||||
|
||||
# Help header, customize to describe your app to your users
|
||||
header: == ${hydra.help.app_name} ==
|
||||
|
||||
footer: |-
|
||||
Powered by Hydra (https://hydra.cc)
|
||||
Use --hydra-help to view Hydra specific help
|
||||
|
||||
template: |-
|
||||
${hydra.help.header}
|
||||
|
||||
pyannote-audio-eval protocol={protocol_name}
|
||||
subset={test | development | train}
|
||||
model={path_to_pretrained_model}
|
||||
warm_up={warm_up_duration_in_seconds}
|
||||
|
||||
${hydra.help.footer}
|
||||
@@ -1,6 +1,6 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2020-2021 CNRS
|
||||
# Copyright (c) 2020-2022 CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -42,7 +42,7 @@ from torch_audiomentations.utils.config import from_dict as get_augmentation
|
||||
from pyannote.database import FileFinder, get_protocol
|
||||
|
||||
|
||||
@hydra.main(config_path="config", config_name="config")
|
||||
@hydra.main(config_path="train_config", config_name="config")
|
||||
def train(cfg: DictConfig) -> Optional[float]:
|
||||
|
||||
# make sure to set the random seed before the instantiation of Trainer
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2020-2021 CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
File renamed without changes.
@@ -0,0 +1,31 @@
|
||||
# @package _group_
|
||||
|
||||
run:
|
||||
dir: ${protocol}/${now:%Y-%m-%dT%H:%M:%S.%fZ}
|
||||
|
||||
sweep:
|
||||
dir: ${protocol}/${now:%Y-%m-%dT%H:%M:%S.%fZ}
|
||||
subdir: ${hydra.job.num}
|
||||
|
||||
output_subdir: ""
|
||||
|
||||
help:
|
||||
app_name: pyannote-audio-train
|
||||
|
||||
# Help header, customize to describe your app to your users
|
||||
header: == ${hydra.help.app_name} ==
|
||||
|
||||
footer: |-
|
||||
Powered by Hydra (https://hydra.cc)
|
||||
Use --hydra-help to view Hydra specific help
|
||||
|
||||
template: |-
|
||||
${hydra.help.header}
|
||||
|
||||
pyannote-audio-train protocol={protocol_name}
|
||||
+task={task} task.param=...
|
||||
+model={model} model.param=...
|
||||
optimizer={optimizer} optimizer.param=...
|
||||
scheduler={scheduler} scheduler.param=...
|
||||
|
||||
${hydra.help.footer}
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,13 @@
|
||||
# @package _group_
|
||||
_target_: pyannote.audio.models.segmentation.PyanNet
|
||||
sincnet:
|
||||
stride: 10
|
||||
lstm:
|
||||
hidden_size: 128
|
||||
num_layers: 2
|
||||
bidirectional: true
|
||||
monolithic: true
|
||||
dropout: 0.5
|
||||
linear:
|
||||
hidden_size: 128
|
||||
num_layers: 2
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Reference in new issue
Block a user