mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
chore: remove prodigy support (#1469)
This commit is contained in:
1 parent
d169fd3770
commit
76d86fa97a
17 files changed
-9883
No files matched your search
@@ -74,7 +74,6 @@ pip install -qq https://github.com/pyannote/pyannote-audio/archive/refs/heads/de
|
||||
- 2021-08-05 > ["Streaming voice activity detection with pyannote.audio"](https://herve.niderb.fr/fastpages/2021/08/05/Streaming-voice-activity-detection-with-pyannote.html)
|
||||
- Miscellaneous
|
||||
- [Training with `pyannote-audio-train` command line tool](tutorials/training_with_cli.md)
|
||||
- [Annotating your own data with Prodigy](tutorials/prodigy.md)
|
||||
- [Speaker verification](tutorials/speaker_verification.ipynb)
|
||||
- Visualization and debugging
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 135 KiB |
@@ -1,367 +0,0 @@
|
||||
// The MIT License (MIT)
|
||||
//
|
||||
// Copyright (c) 2021 CNRS
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
// Can't create constant because prodigy reload this file every batch
|
||||
var currentRegion = 0;
|
||||
var regions = null;
|
||||
var ids = null;
|
||||
var refresh = true;
|
||||
|
||||
var left = 'ArrowLeft';
|
||||
var right = 'ArrowRight';
|
||||
var startR = 'Shift';
|
||||
var endR = 'Control';
|
||||
|
||||
var PRECISION = (prodigy.config.precision / 1000);
|
||||
var BEEP = prodigy.config.beep;
|
||||
var EXCERPT = 1;
|
||||
|
||||
var keysMap = {};
|
||||
|
||||
/**
|
||||
* Handle web audio for beep
|
||||
* @see beep()
|
||||
*/
|
||||
var audioCtx = new(window.AudioContext || window.webkitAudioContext)();
|
||||
|
||||
/**
|
||||
* Makes sure that the document is loaded before executing waitForElement()
|
||||
* Add a small timeout if wavesurfer is already defined (useful when a new batch is coming),
|
||||
* the time for prodigy to update wavesurfer.
|
||||
* @see waitForElement()
|
||||
*/
|
||||
if(document.readyState !== 'loading') {
|
||||
if(typeof window.wavesurfer !== "undefined"){
|
||||
setTimeout(waitForElement,25);
|
||||
}else{
|
||||
waitForElement();
|
||||
}
|
||||
} else {
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
waitForElement();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare if a region is before or after another one
|
||||
* Useful to sort global variable 'ids'
|
||||
* @see reloadWave()
|
||||
* @param {region1} Region object
|
||||
* @param {region2} Region Object
|
||||
* @return {number} -1 if region1 start before, 1 otherwise, 0 if the identical
|
||||
*/
|
||||
function compare(region1, region2){
|
||||
if(region1.start < region2.start){
|
||||
return -1;
|
||||
}else if (region1.start > region2.start){
|
||||
return 1;
|
||||
}else{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulate a click on prodigy's label radio button
|
||||
* Only useful for "review recipe"
|
||||
* Note: might break with future versions of Prodigy
|
||||
* @param {label} Label to click on
|
||||
*/
|
||||
function clickOnLabel(label){
|
||||
document.querySelector("input[type=radio][value=\'"+label+"\']").click()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update prodigy.content with all regions from window.wavesurfer.regions.list (thus, all regions that can be seen in the interface)
|
||||
* Discuss in this issue :
|
||||
* https://support.prodi.gy/t/weird-interaction-between-window-prodigy-update-and-wavesurfer/5450
|
||||
*/
|
||||
function updateContent(){
|
||||
var regions = window.wavesurfer.regions.list;
|
||||
var content = [];
|
||||
for (var id in regions){
|
||||
var region = regions[id];
|
||||
content.push({start : region.start, end : region.end, label : region.label, id : region.id, color : region.color});
|
||||
}
|
||||
window.prodigy.update({audio_spans : content});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a beep sound from scratch
|
||||
* You can adjust the gain, frequency (here its A 440) and duration
|
||||
*/
|
||||
function beep() {
|
||||
if(BEEP){
|
||||
var oscillator = audioCtx.createOscillator();
|
||||
var gainNode = audioCtx.createGain();
|
||||
|
||||
oscillator.connect(gainNode);
|
||||
gainNode.connect(audioCtx.destination);
|
||||
|
||||
gainNode.gain.value = 0.1;
|
||||
oscillator.frequency.value = 440;
|
||||
oscillator.type = "square";
|
||||
|
||||
oscillator.start();
|
||||
|
||||
setTimeout(
|
||||
function() {
|
||||
oscillator.stop();
|
||||
},
|
||||
150 // FIXME: should depend on the value of "precision"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Change CSS style for the selected region
|
||||
* @param {e} Region object
|
||||
*/
|
||||
function activeRegion(e){
|
||||
e.element.style.borderTop = "3px solid";
|
||||
e.element.style.borderBottom = "3px solid";
|
||||
}
|
||||
|
||||
/**
|
||||
* @see activeRegion()
|
||||
* Undo CSS change
|
||||
* @param {e} Region object
|
||||
*/
|
||||
function deactiveRegion(e){
|
||||
e.element.style.borderTop = "";
|
||||
e.element.style.borderBottom = "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Update global variables 'regions' and 'ids' with regions in window.wavesurfer.regions.list
|
||||
* Put the first one as "active" and update the variable currentRegion
|
||||
*/
|
||||
function reloadWave(){
|
||||
regions = window.wavesurfer.regions.list;
|
||||
ids = Object.values(regions);
|
||||
ids.sort(compare);
|
||||
if(ids.length > 0){
|
||||
currentRegion = 0;
|
||||
activeRegion(ids[0]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Switch selected region
|
||||
* Update var currentRegion
|
||||
* Place wavesurfer cursor at the beginning of the new region or the beginning of the file if it's a new prodigy task
|
||||
* @see activeRegion() / deactiveRegion()
|
||||
* @param {ids} Ids of the region to be selected
|
||||
*/
|
||||
function switchCurrent(newId){
|
||||
if(ids.length > 0){
|
||||
deactiveRegion(ids[currentRegion]);
|
||||
currentRegion = newId;
|
||||
activeRegion(ids[newId])
|
||||
if(refresh){
|
||||
window.wavesurfer.seekTo(0);
|
||||
}else{
|
||||
var time = (ids[currentRegion].start) / (window.wavesurfer.getDuration());
|
||||
window.wavesurfer.seekTo(time);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle wavesurfer regions
|
||||
* Add event listener to some wavesurfer event
|
||||
*/
|
||||
function waitForElement(){
|
||||
if(typeof window.wavesurfer !== "undefined"){
|
||||
reloadWave();
|
||||
// Select created region or the first one if it's a new task
|
||||
window.wavesurfer.on('region-created', function(e){
|
||||
setTimeout(function(){
|
||||
if(ids.length > 0) deactiveRegion(ids[currentRegion]);
|
||||
reloadWave();
|
||||
if(refresh){
|
||||
switchCurrent(0);
|
||||
}else{
|
||||
switchCurrent(ids.indexOf(e));
|
||||
}
|
||||
}, 5);
|
||||
});
|
||||
// Change region label (by remove the old one and create a new one with proper label)
|
||||
window.wavesurfer.on('region-dblclick',function(e){
|
||||
re = window.wavesurfer.addRegion({'start' : e.start,'end' : e.end});
|
||||
e.remove();
|
||||
window.wavesurfer.fireEvent('region-update-end',re);
|
||||
});
|
||||
// Select region on click
|
||||
window.wavesurfer.on('region-click',function(e){
|
||||
switchCurrent(ids.indexOf(e));
|
||||
});
|
||||
// Beep when region end
|
||||
window.wavesurfer.on('region-out',function(e){
|
||||
beep();
|
||||
});
|
||||
// @see updateContent()
|
||||
window.wavesurfer.on('region-update-end', function(e){
|
||||
updateContent();
|
||||
});
|
||||
// @see updateContent()
|
||||
// Switch selected region when deleted
|
||||
window.wavesurfer.on('region-removed',function(e){
|
||||
updateContent();
|
||||
if(currentRegion == (ids.length - 1)){
|
||||
var newId = 0;
|
||||
}else{
|
||||
var newId = currentRegion;
|
||||
}
|
||||
reloadWave();
|
||||
if(ids.length > 0) switchCurrent(newId);
|
||||
});
|
||||
}else{
|
||||
setTimeout(waitForElement, 250);
|
||||
}
|
||||
}
|
||||
|
||||
// Check if it's a new prodigy task
|
||||
document.addEventListener('prodigyanswer', e => {
|
||||
refresh = true;
|
||||
})
|
||||
|
||||
/**
|
||||
* Keyboard controller
|
||||
* | Key 1 | Key 2 | Command |
|
||||
* | ------------- | ------------- | ------------ |
|
||||
* | Arrows left/right | [W] | Move Cursor [speed up] |
|
||||
* | Shift | Arrows left/right | Change start of current segment |
|
||||
* | Control | Arrows left/right | Change end of current segment |
|
||||
* | Arrows up/down | | Change current segment to the next/precedent one |
|
||||
* | Shift | Arrows up/[down] | Create [or remove] segment |
|
||||
* | Backspace | | Remove current segment |
|
||||
*/
|
||||
document.querySelector('#root').onkeydown = document.querySelector('#root').onkeyup = function(e){
|
||||
e = e || event;
|
||||
keysMap[e.key] = e.type == 'keydown';
|
||||
var pos = window.wavesurfer.getCurrentTime();
|
||||
var audioEnd = window.wavesurfer.getDuration();
|
||||
var region = ids[currentRegion];
|
||||
refresh = false;
|
||||
|
||||
// If Left is pressed
|
||||
if(keysMap[left] && !keysMap[right]){
|
||||
// If Shift is pressed
|
||||
if(keysMap[startR] && !keysMap[endR]){
|
||||
// Shortens start if possible
|
||||
if((region.start - PRECISION) <= 0){
|
||||
region.update({'start' : 0});
|
||||
window.wavesurfer.fireEvent('region-update-end',region);
|
||||
window.wavesurfer.play(0, region.end);
|
||||
}else{
|
||||
region.update({'start' : region.start - PRECISION });
|
||||
window.wavesurfer.fireEvent('region-update-end',region);
|
||||
window.wavesurfer.play(region.start, region.end);
|
||||
}
|
||||
// If Ctrl is pressed
|
||||
}else if(keysMap[endR] && !keysMap[startR]){
|
||||
var startTime = region.end - EXCERPT;
|
||||
if(startTime < region.start) startTime = region.start;
|
||||
// Shortens end if possible
|
||||
if((region.end - PRECISION) > region.start){
|
||||
region.update({'end' : region.end - PRECISION });
|
||||
window.wavesurfer.fireEvent('region-update-end',region);
|
||||
window.wavesurfer.play(startTime, region.end);
|
||||
}
|
||||
}else{
|
||||
// Else change cursor position
|
||||
// Speed up naviguation if W is pressed
|
||||
if(keysMap['w']){
|
||||
var time = (pos - PRECISION*2) / audioEnd;
|
||||
}else{
|
||||
var time = (pos - PRECISION) / audioEnd;
|
||||
}
|
||||
if(time < 0) time = 0;
|
||||
window.wavesurfer.pause();
|
||||
window.wavesurfer.seekTo(time);
|
||||
}
|
||||
// If Right is pressed
|
||||
}else if(keysMap[right] && !keysMap[left]){
|
||||
// If Shift is pressed
|
||||
if(keysMap[startR] && !keysMap[endR]){
|
||||
// Extend start if possible
|
||||
if(region.start + PRECISION < region.end){
|
||||
region.update({'start' : region.start + PRECISION });
|
||||
window.wavesurfer.fireEvent('region-update-end',region);
|
||||
window.wavesurfer.play(region.start, region.end);
|
||||
}
|
||||
// If Ctrl is pressed
|
||||
}else if(keysMap[endR] && !keysMap[startR]){
|
||||
// Extend end if possible (while keep playing the audio)
|
||||
if(!window.wavesurfer.isPlaying()){
|
||||
var startTime = region.end - EXCERPT;
|
||||
if(startTime < region.start) startTime = region.start;
|
||||
}else{
|
||||
var startTime = pos;
|
||||
}
|
||||
if((region.end + PRECISION) >= audioEnd){
|
||||
region.update({'end' : audioEnd });
|
||||
window.wavesurfer.fireEvent('region-update-end',region);
|
||||
}else{
|
||||
region.update({'end' : region.end + PRECISION });
|
||||
window.wavesurfer.fireEvent('region-update-end',region);
|
||||
}
|
||||
window.wavesurfer.play(startTime, region.end);
|
||||
}else{
|
||||
// Else change cursor position
|
||||
// Speed up naviguation if W is pressed
|
||||
if(keysMap['w']){
|
||||
var time = (pos + PRECISION*2) / audioEnd;
|
||||
}else{
|
||||
var time = (pos + PRECISION) / audioEnd;
|
||||
}
|
||||
if(time > 1) time = 1;
|
||||
window.wavesurfer.pause();
|
||||
window.wavesurfer.seekTo(time);
|
||||
}
|
||||
// If Up and shift is pressed : new region
|
||||
}else if (keysMap['ArrowUp'] && keysMap['Shift']){
|
||||
var fin = pos + 1;
|
||||
if(fin > audioEnd) fin = audioEnd;
|
||||
re = window.wavesurfer.addRegion({'start' : pos,'end' : fin});
|
||||
window.wavesurfer.fireEvent('region-update-end',re);
|
||||
// If Down and Shift or Backspace: delete region
|
||||
// Check backspace for diarization text field
|
||||
}else if(keysMap['Backspace'] || (keysMap['ArrowDown'] && keysMap['Shift'])){
|
||||
ids[currentRegion].remove();
|
||||
// If Up/Down @see switchCurrent
|
||||
}else if(keysMap['ArrowUp']){
|
||||
if(currentRegion == (ids.length - 1)){
|
||||
switchCurrent(0);
|
||||
}else{
|
||||
switchCurrent(currentRegion + 1);
|
||||
}
|
||||
}else if(keysMap['ArrowDown']){
|
||||
if(currentRegion == 0){
|
||||
switchCurrent(ids.length - 1);
|
||||
}else{
|
||||
switchCurrent(currentRegion - 1);
|
||||
}
|
||||
}else if(keysMap['u']){
|
||||
reloadWave();
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<h1>Commands</h1>
|
||||
</head>
|
||||
<body>
|
||||
You have to mark the speech moments with bounding boxes.
|
||||
<br/>
|
||||
<img src='data:image/png;base64,{IMAGE}' with="812" height="284">
|
||||
<br/><br/>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Key 1</th>
|
||||
<th>Key 2</th>
|
||||
<th>Command</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Arrows left/right</td>
|
||||
<td>[W]</td>
|
||||
<td>Move Cursor [speed up]</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Shift</td>
|
||||
<td>Arrow left/right</td>
|
||||
<td>Change start of current segment</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Control</td>
|
||||
<td>Arrow left/right</td>
|
||||
<td>Change end of current segment</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Arrows up/down</td>
|
||||
<td></td>
|
||||
<td>Change current segment to the next/precedent one</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Shift</td>
|
||||
<td>Arrow up/[down]</td>
|
||||
<td>Create or [remove] segment</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Backspace</td>
|
||||
<td></td>
|
||||
<td>Remove current segment</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Spacebar</td>
|
||||
<td></td>
|
||||
<td>Play/pause audio</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>U</td>
|
||||
<td></td>
|
||||
<td>Undo all actions</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Escape</td>
|
||||
<td></td>
|
||||
<td>Ignore this sample</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Enter</td>
|
||||
<td></td>
|
||||
<td>Validate annotation</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,41 +0,0 @@
|
||||
.pyannote-region {
|
||||
left: 1px;
|
||||
color: #000;
|
||||
bottom: 0;
|
||||
display: block;
|
||||
padding: 0 3px;
|
||||
z-index: 10;
|
||||
position: absolute;
|
||||
font-size: 0.7em;
|
||||
background: #ffe184;
|
||||
font-weight: bold;
|
||||
font-family: "Roboto Condensed", "Arial Narrow", sans-serif;
|
||||
user-select: none;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.title-wave {
|
||||
left: 1px;
|
||||
color: #000;
|
||||
display: block;
|
||||
padding: 0 3px;
|
||||
z-index: 10;
|
||||
position: absolute;
|
||||
font-size: 0.7em;
|
||||
background: #ddd;
|
||||
font-weight: bold;
|
||||
font-family: "Lato", "Trebuchet MS", Roboto, Helvetica, Arial, sans-serif;
|
||||
user-select: none;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.prodigy-content{ white-space: inherit }
|
||||
|
||||
wave{ overflow: hidden !important;}
|
||||
|
||||
.legend { list-style: none; }
|
||||
.legend li { float: left; margin-right: 10px; }
|
||||
.legend span { border: 1px solid #ccc; float: left; width: 30px; height: 30px; margin: 2px; }
|
||||
.legend .falsealarm { background-color: #9932cc33; }
|
||||
.legend .confusion { background-color: #ff634733; }
|
||||
.legend .misseddetection { background-color: #00ffff33; }
|
||||
@@ -1,3 +0,0 @@
|
||||
<audio id='track'>
|
||||
<source src={{audio}} type='audio/x-wav'/>
|
||||
</audio>
|
||||
@@ -1,169 +0,0 @@
|
||||
# The MIT License (MIT)
|
||||
#
|
||||
# Copyright (c) 2021 CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import base64
|
||||
import io
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Text
|
||||
|
||||
import numpy as np
|
||||
import scipy.io.wavfile
|
||||
from prodigy.components.loaders import Audio as ProdigyAudioLoader
|
||||
from pyannote.core import Annotation, Segment, SlidingWindow
|
||||
|
||||
from pyannote.audio import Audio
|
||||
|
||||
|
||||
class AudioForProdigy(Audio):
|
||||
def __init__(self):
|
||||
super().__init__(sample_rate=16000, mono="downmix")
|
||||
|
||||
def crop(self, path: Path, excerpt: Segment) -> Text:
|
||||
waveform, _ = super().crop(path, excerpt)
|
||||
waveform = waveform.numpy().T
|
||||
waveform /= np.max(np.abs(waveform)) + 1e-8
|
||||
with io.BytesIO() as content:
|
||||
scipy.io.wavfile.write(content, self.sample_rate, waveform)
|
||||
content.seek(0)
|
||||
b64 = base64.b64encode(content.read()).decode()
|
||||
b64 = f"data:audio/x-wav;base64,{b64}"
|
||||
return b64
|
||||
|
||||
|
||||
def to_audio_spans(annotation: Annotation, focus: Segment = None) -> Dict:
|
||||
"""Convert pyannote.core.Annotation to Prodigy's audio_spans
|
||||
|
||||
Parameters
|
||||
----------
|
||||
annotation : Annotation
|
||||
Annotation with t=0s time origin.
|
||||
focus : Segment, optional
|
||||
When provided, use its start time as audio_spans time origin.
|
||||
|
||||
Returns
|
||||
-------
|
||||
audio_spans : list of dict
|
||||
"""
|
||||
shift = 0.0 if focus is None else focus.start
|
||||
return [
|
||||
{"start": segment.start - shift, "end": segment.end - shift, "label": label}
|
||||
for segment, _, label in annotation.itertracks(yield_label=True)
|
||||
]
|
||||
|
||||
|
||||
def get_audio_spans(
|
||||
annotation: Annotation, excerpt: Segment, excerpt_with_context: Segment = None
|
||||
):
|
||||
|
||||
excerpt_with_context = excerpt_with_context or excerpt
|
||||
shift = excerpt.start - excerpt_with_context.start
|
||||
|
||||
shifted_excerpt = Segment(start=shift, end=shift + excerpt.duration)
|
||||
excerpt_annotation = annotation.crop(shifted_excerpt)
|
||||
|
||||
return [
|
||||
{"start": segment.start - shift, "end": segment.end - shift, "label": label}
|
||||
for segment, _, label in excerpt_annotation.itertracks(yield_label=True)
|
||||
]
|
||||
|
||||
|
||||
def remove_audio_before_db(examples: List[Dict]) -> List[Dict]:
|
||||
"""Remove (potentially heavy) 'audio' key from examples
|
||||
|
||||
Parameters
|
||||
----------
|
||||
examples : list of dict
|
||||
Examples.
|
||||
|
||||
Returns
|
||||
-------
|
||||
examples : list of dict
|
||||
Examples with 'audio' key removed.
|
||||
"""
|
||||
for eg in examples:
|
||||
if "audio" in eg:
|
||||
del eg["audio"]
|
||||
|
||||
return examples
|
||||
|
||||
|
||||
def source_to_files(source: Path) -> List[Dict]:
|
||||
"""
|
||||
Convert a directory or a file path to a list of files object for prodigy
|
||||
"""
|
||||
if source.is_dir():
|
||||
files = ProdigyAudioLoader(source)
|
||||
else:
|
||||
name = source.stem
|
||||
files = [{"path": source, "text": name, "meta": {"file": source}}]
|
||||
|
||||
return files
|
||||
|
||||
|
||||
def get_chunks(source: Path, chunk_duration: Optional[float] = None):
|
||||
|
||||
files = source_to_files(source)
|
||||
audio = Audio(mono="downmix")
|
||||
|
||||
for file in files:
|
||||
|
||||
duration = audio.get_duration(file["path"])
|
||||
|
||||
if (chunk_duration is None) or (duration < chunk_duration):
|
||||
chunks = [Segment(0, duration)]
|
||||
else:
|
||||
windows = SlidingWindow(
|
||||
start=0.0, duration=chunk_duration, step=chunk_duration
|
||||
)
|
||||
chunks = windows(Segment(0, duration), align_last=True)
|
||||
|
||||
for chunk in chunks:
|
||||
yield file, chunk
|
||||
|
||||
|
||||
def before_db(examples):
|
||||
"""Post-process examples before sending them to the database
|
||||
|
||||
1. Remove "audio" key as it is very heavy and can easily be retrieved from other keys
|
||||
2. Shift Prodigy/wavesurfer chunk-based audio spans so that their timing are file-based.
|
||||
"""
|
||||
|
||||
for eg in examples:
|
||||
|
||||
# 1. remove "audio" key as it can be retrieved from "chunk" and "path" keys
|
||||
if "audio" in eg:
|
||||
del eg["audio"]
|
||||
|
||||
# 2. shift audio spans
|
||||
chunk_start = eg["chunk"]["start"]
|
||||
audio_spans_keys = [key for key in eg if "audio_spans" in key]
|
||||
for key in audio_spans_keys:
|
||||
eg[key] = [
|
||||
{
|
||||
"start": span["start"] + chunk_start,
|
||||
"end": span["end"] + chunk_start,
|
||||
"label": span["label"],
|
||||
}
|
||||
for span in eg[key]
|
||||
]
|
||||
|
||||
return examples
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,231 +0,0 @@
|
||||
// The MIT License (MIT)
|
||||
//
|
||||
// Copyright (c) 2021 CNRS
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
var numberAnnotations = 2;
|
||||
var colorList = [];
|
||||
var hex2rgba = (hex, alpha = 0.2) => {
|
||||
const [r, g, b] = hex.match(/\w\w/g).map(x => parseInt(x, 16));
|
||||
return `rgba(${r},${g},${b},${alpha})`;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Makes sure that the document is loaded before executing waitForElement()
|
||||
* @see waitForElement()
|
||||
*/
|
||||
if(document.readyState !== 'loading') {
|
||||
waitForElement();
|
||||
} else {
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
waitForElement();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove label on region (created by prodigy)
|
||||
* @param(r) region object
|
||||
* @see waitForElement()
|
||||
*/
|
||||
function removeLabel(r){
|
||||
s = r.start;
|
||||
e = r.end;
|
||||
c = r.color;
|
||||
r.remove();
|
||||
window.wavesurfer.addRegion({"start" : s, "end" :e, 'color' : c, "resize": false, "drag": false});
|
||||
}
|
||||
|
||||
/**
|
||||
* Add reference and hypothesis regions on the corresponding wavesurfer instance
|
||||
*/
|
||||
function loadRegions(){
|
||||
for(var i=0; i < numberAnnotations;i++){
|
||||
(i == 0)? (regions = window.prodigy.content.reference) : (regions = window.prodigy.content.hypothesis)
|
||||
for (region in regions){
|
||||
var label = regions[region]['label'];
|
||||
if (!(label in colorList)){
|
||||
colorList.push(label);
|
||||
}
|
||||
var id = colorList.indexOf(label) % prodigy.config.custom_theme.palettes.audio.length;
|
||||
var color = prodigy.config.custom_theme.palettes.audio[id];
|
||||
color = hex2rgba(color);
|
||||
var re = window['wavesurfer'+i].addRegion({'start' : regions[region]['start'],'end' : regions[region]['end'],'color' : color, 'resize' : false, 'drag' : false, "attributes": {"label":regions[region]['label']}});
|
||||
addRegionLabel(re,label);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create audio url for reference and hypothesis wavesurfer instance
|
||||
* @see loadWave()
|
||||
* @see LoadTrack()
|
||||
*/
|
||||
async function createURL(){
|
||||
var blob;
|
||||
var track = document.querySelector('#track');
|
||||
var src = track.children[0].src
|
||||
var blob = await (await fetch(src)).blob()
|
||||
var objectURL = URL.createObjectURL(blob);
|
||||
return objectURL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the same wavesurfer object as on Prodigy interface for reference and hypothesis
|
||||
* Those objects are muted
|
||||
*/
|
||||
async function loadWave(){
|
||||
var objectURL = await createURL();
|
||||
var wdict = {
|
||||
container: window.wavesurfer.container,
|
||||
audioRate: 1,
|
||||
autoCenter: true,
|
||||
autoCenterImmediately: false,
|
||||
backend: "WebAudio",
|
||||
barGap: 2,
|
||||
barHeight: 1,
|
||||
barMinHeight: null,
|
||||
barRadius: 2,
|
||||
barWidth: 0,
|
||||
cursorColor: "#333",
|
||||
cursorWidth: 1,
|
||||
fillParent: true,
|
||||
forceDecode: false,
|
||||
height: 128,
|
||||
hideScrollbar: false,
|
||||
interact: false,
|
||||
loopSelection: true,
|
||||
maxCanvasWidth: 4000,
|
||||
mediaControls: false,
|
||||
mediaType: "audio",
|
||||
normalize: false,
|
||||
partialRender: false,
|
||||
pixelRatio: 2,
|
||||
progressColor: "#583fcf",
|
||||
waveColor: "violet",
|
||||
removeMediaElementOnDestroy: true,
|
||||
responsive: false,
|
||||
scrollParent: true,
|
||||
skipLength: 2,
|
||||
splitChannels: false,
|
||||
plugins: [
|
||||
WaveSurfer.regions.create({})
|
||||
]
|
||||
};
|
||||
for(var i=0; i < numberAnnotations;i++){
|
||||
window['wavesurfer'+i] = WaveSurfer.create(wdict);
|
||||
window['wavesurfer'+i].load(objectURL);
|
||||
window['wavesurfer'+i].setMute(true);
|
||||
}
|
||||
var l = document.querySelector('wave').appendChild(document.createElement("span"));
|
||||
l.textContent= "Errors";
|
||||
l.className = "title-wave";
|
||||
const nodeList = document.querySelectorAll('wave ~ wave');
|
||||
for(var i = 0; i < nodeList.length;i++){
|
||||
nodeList[i].style.marginTop = "30px";
|
||||
nodeList[i].style.backgroundColor = "#0000000a";
|
||||
l = nodeList[i].appendChild(document.createElement("span"));
|
||||
(i == 0)? (l.textContent = "Reference") : (l.textContent = "Hypothesis")
|
||||
l.className = "title-wave";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle wavesurfer
|
||||
* Tracking audio playback for reference and hypothesis wavesurfer
|
||||
*/
|
||||
async function waitForElement(){
|
||||
if((typeof window.wavesurfer !== "undefined") && (document.querySelector('#track') !== null)){
|
||||
window.wavesurfer.on('region-created', function(e){
|
||||
setTimeout(function(){
|
||||
if("label" in e){
|
||||
removeLabel(e);
|
||||
}},5);
|
||||
})
|
||||
window.wavesurfer.on('audioprocess', function(e){
|
||||
var time = e / window.wavesurfer.getDuration();
|
||||
for(var i=0; i < numberAnnotations;i++){
|
||||
window['wavesurfer'+i].seekAndCenter(time);
|
||||
}
|
||||
});
|
||||
window.wavesurfer.on('seek', function(e){
|
||||
for(var i=0; i < numberAnnotations;i++){
|
||||
window['wavesurfer'+i].seekTo(e);
|
||||
}
|
||||
});
|
||||
window.wavesurfer.on('zoom', function(e){
|
||||
for(var i=0; i < numberAnnotations;i++){
|
||||
window['wavesurfer'+i].zoom(e);
|
||||
}
|
||||
});
|
||||
window.wavesurfer.on('finish', function(e){
|
||||
for(var i=0; i < numberAnnotations;i++){
|
||||
window['wavesurfer'+i].seekTo(1);
|
||||
}
|
||||
});
|
||||
window.wavesurfer.on('pause', function(e){
|
||||
for(var i=0; i < numberAnnotations;i++){
|
||||
window['wavesurfer'+i].pause();
|
||||
}
|
||||
});
|
||||
await loadWave();
|
||||
loadRegions();
|
||||
}else{
|
||||
setTimeout(waitForElement, 250);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reload audio (when there is a new task)
|
||||
* @see prodigyanswer
|
||||
*/
|
||||
async function loadTrack(){
|
||||
if(document.querySelector('#track') !== null){
|
||||
var objectURL = await createURL();
|
||||
for(var i=0; i < numberAnnotations;i++){
|
||||
window['wavesurfer'+i].load(objectURL);
|
||||
window['wavesurfer'+i].clearRegions();
|
||||
}
|
||||
loadRegions();
|
||||
}else{
|
||||
setTimeout(loadTrack, 250);
|
||||
}
|
||||
}
|
||||
|
||||
// Check if it's a new prodigy task
|
||||
document.addEventListener('prodigyanswer', async() => {
|
||||
loadTrack();
|
||||
});
|
||||
|
||||
/**
|
||||
* Add CSS label to reference and hypothesis region
|
||||
* @see loadRegions()
|
||||
* @param(r) region object
|
||||
* @param(l) label
|
||||
*/
|
||||
function addRegionLabel(r,label){
|
||||
var s = r.element
|
||||
var l = s.appendChild(document.createElement("span"))
|
||||
l.textContent = label,
|
||||
l.className = "pyannote-region",
|
||||
l.style.color = "rgb(0, 0, 0)",
|
||||
l.style.background = r.color,
|
||||
r.label = label;
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
<ul class="legend">
|
||||
<li><span class="confusion"></span>Confusion</li>
|
||||
<li><span class="falsealarm"></span> False Alarm</li>
|
||||
<li><span class="misseddetection"></span>Missed Detection</li>
|
||||
</ul>
|
||||
@@ -1,258 +0,0 @@
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List
|
||||
|
||||
import prodigy
|
||||
from prodigy import set_hashes
|
||||
from pyannote.core import Annotation, Segment
|
||||
from pyannote.database import util
|
||||
from pyannote.metrics.errors.identification import IdentificationErrorAnalysis
|
||||
|
||||
from pyannote.audio import Audio
|
||||
from pyannote.audio.pipelines import SpeakerDiarization
|
||||
|
||||
from ..common.utils import (
|
||||
AudioForProdigy,
|
||||
before_db,
|
||||
get_audio_spans,
|
||||
get_chunks,
|
||||
source_to_files,
|
||||
)
|
||||
|
||||
|
||||
def diff_stream(
|
||||
source: Path,
|
||||
reference: dict,
|
||||
hypothesis: dict,
|
||||
list_errors: List,
|
||||
diarization: bool = False,
|
||||
chunk: float = 30.0,
|
||||
min_duration: int = 200,
|
||||
) -> Iterable[Dict]:
|
||||
|
||||
files = source_to_files(source)
|
||||
files_errors = {}
|
||||
for file in files:
|
||||
filename = file["text"]
|
||||
ref = reference[filename]
|
||||
hyp = hypothesis[filename]
|
||||
if diarization:
|
||||
hyp: Annotation = SpeakerDiarization.optimal_mapping(ref, hyp)
|
||||
identificationErrorAnalysis = IdentificationErrorAnalysis()
|
||||
errors = identificationErrorAnalysis.difference(ref, hyp)
|
||||
newLabels = {}
|
||||
for labels in errors.labels():
|
||||
a, b, c = labels
|
||||
newLabels[(a, b, c)] = a
|
||||
errors = errors.rename_labels(newLabels)
|
||||
errors = errors.subset(["correct"], invert=True)
|
||||
if list_errors[0] or list_errors[1] or list_errors[2]:
|
||||
if not list_errors[0]:
|
||||
errors = errors.subset(["false alarm"], invert=True)
|
||||
if not list_errors[1]:
|
||||
errors = errors.subset(["confusion"], invert=True)
|
||||
if not list_errors[2]:
|
||||
errors = errors.subset(["missed detection"], invert=True)
|
||||
clean_errors = Annotation()
|
||||
for segment, track, label in errors.itertracks(yield_label=True):
|
||||
if segment.duration * 1000 > min_duration:
|
||||
clean_errors[segment, track] = label
|
||||
errors = clean_errors
|
||||
files_errors[filename] = errors
|
||||
|
||||
audio_for_prodigy = AudioForProdigy()
|
||||
audio_for_pipeline = Audio(mono="downmix")
|
||||
chunks = get_chunks(source, chunk_duration=chunk)
|
||||
chunks = list(chunks)
|
||||
chunks = sorted(
|
||||
chunks,
|
||||
key=lambda k: max(
|
||||
(
|
||||
files_errors[k[0]["text"]]
|
||||
.crop(k[1], mode="intersection")
|
||||
.label_duration(e)
|
||||
for e in files_errors[k[0]["text"]]
|
||||
.crop(k[1], mode="intersection")
|
||||
.labels()
|
||||
),
|
||||
default=0,
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
for file, excerpt in chunks:
|
||||
path = file["path"]
|
||||
filename = file["text"]
|
||||
text = f"{filename} [{excerpt.start:.1f} - {excerpt.end:.1f}]"
|
||||
duration = audio_for_pipeline.get_duration(path)
|
||||
# load audio excerpt
|
||||
waveform, sample_rate = audio_for_pipeline.crop(path, excerpt, mode="pad")
|
||||
# load audio excerpt for visualization in Prodigy
|
||||
audio = audio_for_prodigy.crop(path, excerpt)
|
||||
|
||||
audio_spans = get_audio_spans(
|
||||
files_errors[filename], excerpt, Segment(0, duration)
|
||||
)
|
||||
ref = get_audio_spans(reference[filename], excerpt, Segment(0, duration))
|
||||
hyp = get_audio_spans(hypothesis[filename], excerpt, Segment(0, duration))
|
||||
|
||||
yield {
|
||||
"path": path,
|
||||
"text": text,
|
||||
"audio": audio,
|
||||
"audio_spans": audio_spans,
|
||||
"reference": ref,
|
||||
"hypothesis": hyp,
|
||||
"chunk": {"start": excerpt.start, "end": excerpt.end},
|
||||
"meta": {
|
||||
"file": filename,
|
||||
"start": f"{excerpt.start:.1f}",
|
||||
"end": f"{excerpt.end:.1f}",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# RTTM files contain one line per speech turn, using the following convention:
|
||||
# SPEAKER {uri} 1 {start_time} {duration} <NA> <NA> {speaker_id} <NA> <NA>
|
||||
# * uri: file identifier (as given by pyannote.database protocols)
|
||||
# * start_time: speech turn start time in seconds
|
||||
# * duration: speech turn duration in seconds
|
||||
# * confidence: confidence score (can be anything, not used for now)
|
||||
# * gender: speaker gender (can be anything, not used for now)
|
||||
# * speaker_id: speaker identifier
|
||||
|
||||
|
||||
@prodigy.recipe(
|
||||
"pyannote.diff",
|
||||
dataset=("Dataset to save annotations to", "positional", None, str),
|
||||
source=(
|
||||
"Path to directory containing audio files whose annotation is to be checked",
|
||||
"positional",
|
||||
None,
|
||||
Path,
|
||||
),
|
||||
reference=(
|
||||
"Path to reference file using the RTTM file format",
|
||||
"positional",
|
||||
None,
|
||||
str,
|
||||
),
|
||||
hypothesis=(
|
||||
"Path to hypothesis file using the RTTM file format",
|
||||
"positional",
|
||||
None,
|
||||
str,
|
||||
),
|
||||
chunk=(
|
||||
"Split long audio files into shorter chunks of that many seconds each",
|
||||
"option",
|
||||
None,
|
||||
float,
|
||||
),
|
||||
min_duration=("Minimum duration of errors in ms", "option", None, int),
|
||||
diarization=(
|
||||
"Optimal one-to-one mapping between reference and hypothesis",
|
||||
"flag",
|
||||
None,
|
||||
bool,
|
||||
),
|
||||
false_alarm=("Display false alarm errors", "flag", None, bool),
|
||||
speaker_confusion=("Display confusion errors", "flag", None, bool),
|
||||
missed_detection=("Display missed detection errors", "flag", None, bool),
|
||||
)
|
||||
def diff(
|
||||
dataset: str,
|
||||
source: Path,
|
||||
reference: str,
|
||||
hypothesis: str,
|
||||
chunk: float = 30.0,
|
||||
min_duration=200,
|
||||
diarization: bool = False,
|
||||
false_alarm: bool = False,
|
||||
speaker_confusion: bool = False,
|
||||
missed_detection: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
|
||||
recipe_dir = Path(__file__).resolve().parent
|
||||
common_dir = recipe_dir.parent / "common"
|
||||
|
||||
controllerDiff = recipe_dir / "controller.js"
|
||||
legend = recipe_dir / "legend.html"
|
||||
wavesurfer = common_dir / "wavesurfer.js"
|
||||
regions = common_dir / "regions.js"
|
||||
html = common_dir / "template.html"
|
||||
css = common_dir / "template.css"
|
||||
|
||||
with open(controllerDiff) as sc_diff, open(wavesurfer) as s_wavesurfer, open(
|
||||
regions
|
||||
) as s_regions, open(html) as f_html, open(legend) as f_legend, open(css) as f_css:
|
||||
script_text = s_wavesurfer.read()
|
||||
script_text += "\n" + s_regions.read()
|
||||
script_text += "\n" + sc_diff.read()
|
||||
templateH = f_html.read()
|
||||
templateL = f_legend.read()
|
||||
templateC = f_css.read()
|
||||
|
||||
ref = util.load_rttm(reference)
|
||||
hyp = util.load_rttm(hypothesis)
|
||||
list_errors = [false_alarm, speaker_confusion, missed_detection]
|
||||
|
||||
hashed_stream = (
|
||||
set_hashes(eg, input_keys=("path", "chunk"))
|
||||
for eg in diff_stream(
|
||||
source,
|
||||
ref,
|
||||
hyp,
|
||||
list_errors,
|
||||
diarization=diarization,
|
||||
chunk=chunk,
|
||||
min_duration=min_duration,
|
||||
)
|
||||
)
|
||||
|
||||
return {
|
||||
"view_id": "blocks",
|
||||
"dataset": dataset,
|
||||
"stream": hashed_stream,
|
||||
"before_db": before_db,
|
||||
"config": {
|
||||
"global_css": templateC,
|
||||
"javascript": script_text,
|
||||
"show_audio_minimap": False,
|
||||
"audio_bar_width": 0,
|
||||
"audio_bar_height": 1,
|
||||
"custom_theme": {
|
||||
"cardMinHeight": 400,
|
||||
"labels": {
|
||||
"false alarm": "#9932cc",
|
||||
"confusion": "#ff6347",
|
||||
"missed detection": "#00ffff",
|
||||
},
|
||||
"palettes": {
|
||||
"audio": [
|
||||
"#ffd700",
|
||||
"#00ffff",
|
||||
"#ff00ff",
|
||||
"#00ff00",
|
||||
"#9932cc",
|
||||
"#00bfff",
|
||||
"#ff7f50",
|
||||
"#66cdaa",
|
||||
],
|
||||
},
|
||||
},
|
||||
"blocks": [
|
||||
{"view_id": "html", "html_template": templateL},
|
||||
{"view_id": "audio"},
|
||||
{"view_id": "html", "html_template": templateH},
|
||||
],
|
||||
"show_audio_timeline": True,
|
||||
"buttons": ["accept", "ignore", "undo"],
|
||||
"keymap": {
|
||||
"accept": ["enter"],
|
||||
"ignore": ["escape"],
|
||||
"undo": ["u"],
|
||||
"playpause": ["space"],
|
||||
},
|
||||
"show_flag": True,
|
||||
},
|
||||
}
|
||||
@@ -1,231 +0,0 @@
|
||||
# The MIT License (MIT)
|
||||
#
|
||||
# Copyright (c) 2021-2022 CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# AUTHORS
|
||||
# Jim Petiot
|
||||
# Hervé Bredin
|
||||
|
||||
import base64
|
||||
import random
|
||||
from collections.abc import Iterator
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from tempfile import mkstemp
|
||||
from typing import Any, Dict, Iterable, List, Union
|
||||
|
||||
import prodigy
|
||||
from prodigy import set_hashes
|
||||
from pyannote.core import Annotation, Segment
|
||||
|
||||
from pyannote.audio import Audio, Pipeline
|
||||
|
||||
from ..common.utils import AudioForProdigy, before_db, get_audio_spans, get_chunks
|
||||
|
||||
|
||||
def stream(
|
||||
pipeline: Pipeline,
|
||||
source: Path,
|
||||
labels: List[str],
|
||||
chunk: float = 30.0,
|
||||
randomize: bool = False,
|
||||
) -> Iterable[Dict]:
|
||||
"""Stream for `pyannote.audio` recipe
|
||||
|
||||
Applies pretrained pipeline and sends the results for manual correction
|
||||
|
||||
Parameters
|
||||
----------
|
||||
pipeline : Pipeline
|
||||
Pretrained pipeline.
|
||||
source : Path
|
||||
Directory containing audio files to process.
|
||||
labels : list of string
|
||||
List of expected pipeline labels.
|
||||
chunk : float, optional
|
||||
Duration of chunks, in seconds. Defaults to 30s.
|
||||
|
||||
Yields
|
||||
------
|
||||
task : dict
|
||||
Prodigy task with the following keys:
|
||||
"path" : path to audio file
|
||||
"chunk" : chunk start and end times
|
||||
"audio" : base64 encoding of audio chunk
|
||||
"text" : chunk identifier "{filename} [{start} {end}]"
|
||||
"audio_spans" : list of audio spans {"start": ..., "end": ..., "label": ...}
|
||||
"audio_spans_original" : deep copy of "audio_spans"
|
||||
"meta" : metadata displayed in Prodigy UI {"file": ..., "start": ..., "end": ...}
|
||||
"config": {"labels": list of labels}
|
||||
"""
|
||||
|
||||
context = getattr(pipeline, "context", 2.5)
|
||||
|
||||
audio_for_prodigy = AudioForProdigy()
|
||||
audio_for_pipeline = Audio(mono="downmix")
|
||||
|
||||
chunks = get_chunks(source, chunk_duration=chunk)
|
||||
if randomize:
|
||||
chunks = list(chunks)
|
||||
random.shuffle(chunks)
|
||||
|
||||
for file, excerpt in chunks:
|
||||
|
||||
path = file["path"]
|
||||
filename = file["text"]
|
||||
text = f"{filename} [{excerpt.start:.1f} - {excerpt.end:.1f}]"
|
||||
|
||||
# load contextualized audio excerpt
|
||||
excerpt_with_context = Segment(
|
||||
start=excerpt.start - context, end=excerpt.end + context
|
||||
)
|
||||
waveform_with_context, sample_rate = audio_for_pipeline.crop(
|
||||
path, excerpt_with_context, mode="pad"
|
||||
)
|
||||
|
||||
# run pipeline on contextualized audio excerpt
|
||||
output: Annotation = pipeline(
|
||||
{"waveform": waveform_with_context, "sample_rate": sample_rate}
|
||||
)
|
||||
|
||||
# crop, shift, and format output for visualization in Prodigy
|
||||
audio_spans = get_audio_spans(
|
||||
output, excerpt, excerpt_with_context=excerpt_with_context
|
||||
)
|
||||
|
||||
# load audio excerpt for visualization in Prodigy
|
||||
audio = audio_for_prodigy.crop(path, excerpt)
|
||||
|
||||
yield {
|
||||
"path": path,
|
||||
"text": text,
|
||||
"audio": audio,
|
||||
"audio_spans": audio_spans,
|
||||
"audio_spans_original": deepcopy(audio_spans),
|
||||
"chunk": {"start": excerpt.start, "end": excerpt.end},
|
||||
"config": {"labels": sorted(set(output.labels()) | set(labels))},
|
||||
"meta": {
|
||||
"file": filename,
|
||||
"start": f"{excerpt.start:.1f}",
|
||||
"end": f"{excerpt.end:.1f}",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@prodigy.recipe(
|
||||
"pyannote.audio",
|
||||
dataset=("Dataset to save annotations to", "positional", None, str),
|
||||
source=(
|
||||
"Path to directory containing audio files to annotate",
|
||||
"positional",
|
||||
None,
|
||||
Path,
|
||||
),
|
||||
pipeline=(
|
||||
"Pretrained pipeline (name on Huggingface Hub, path to YAML file)",
|
||||
"positional",
|
||||
None,
|
||||
str,
|
||||
),
|
||||
chunk=(
|
||||
"Split long audio files into shorter chunks of that many seconds each",
|
||||
"option",
|
||||
None,
|
||||
float,
|
||||
),
|
||||
num_classes=(
|
||||
"Set maximum number of classes for pipelines whose number of classes is not predefined (e.g. pyannote/speaker-diarization)",
|
||||
"option",
|
||||
None,
|
||||
int,
|
||||
),
|
||||
precision=("Keyboard temporal precision, in milliseconds.", "option", None, int),
|
||||
beep=(
|
||||
"Beep when the player reaches the end of a region.",
|
||||
"flag",
|
||||
None,
|
||||
bool,
|
||||
),
|
||||
)
|
||||
def pipeline(
|
||||
dataset: str,
|
||||
source: Path,
|
||||
pipeline: Union[str, Iterable[dict]],
|
||||
chunk: float = 10.0,
|
||||
num_classes: int = 4,
|
||||
precision: int = 200,
|
||||
beep: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
|
||||
pipeline = Pipeline.from_pretrained(pipeline, use_auth_token=True)
|
||||
classes = pipeline.classes()
|
||||
|
||||
if isinstance(classes, Iterator):
|
||||
labels = [x for _, x in zip(range(num_classes), classes)]
|
||||
else:
|
||||
labels = classes
|
||||
|
||||
recipe_dir = Path(__file__).resolve().parent
|
||||
common_dir = recipe_dir.parent / "common"
|
||||
controller_js = common_dir / "controller.js"
|
||||
with open(controller_js) as txt:
|
||||
javascript = txt.read()
|
||||
|
||||
# TODO: improve this part
|
||||
template = common_dir / "instructions.html"
|
||||
png = common_dir / "commands.png"
|
||||
_, instructions_html = mkstemp(text=True)
|
||||
with open(instructions_html, "w") as instructions_f, open(
|
||||
template, "r"
|
||||
) as fp_tpl, open(png, "rb") as fp_png:
|
||||
b64 = base64.b64encode(fp_png.read()).decode("utf-8")
|
||||
instructions_f.write(fp_tpl.read().replace("{IMAGE}", b64))
|
||||
|
||||
hashed_stream = (
|
||||
set_hashes(eg, input_keys=("path", "chunk"))
|
||||
for eg in stream(pipeline, source, labels, chunk=chunk, randomize=False)
|
||||
)
|
||||
|
||||
return {
|
||||
"view_id": "audio_manual",
|
||||
"dataset": dataset,
|
||||
"stream": hashed_stream,
|
||||
"before_db": before_db,
|
||||
"config": {
|
||||
"javascript": javascript,
|
||||
"instructions": instructions_html,
|
||||
"buttons": ["accept", "ignore", "undo"],
|
||||
"keymap": {
|
||||
"accept": ["enter"],
|
||||
"ignore": ["escape"],
|
||||
"undo": ["u"],
|
||||
"playpause": ["space"],
|
||||
},
|
||||
"show_audio_minimap": False,
|
||||
"audio_autoplay": True,
|
||||
"audio_bar_width": 0,
|
||||
"audio_bar_height": 1,
|
||||
"show_flag": True,
|
||||
"labels": labels,
|
||||
"precision": precision,
|
||||
"beep": beep,
|
||||
},
|
||||
}
|
||||
@@ -1,228 +0,0 @@
|
||||
// The MIT License (MIT)
|
||||
//
|
||||
// Copyright (c) 2021 CNRS
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
var numberAnnotations = prodigy.config.number_annotations;
|
||||
|
||||
var colorList = [];
|
||||
var hex2rgba = (hex, alpha = 0.2) => {
|
||||
const [r, g, b] = hex.match(/\w\w/g).map(x => parseInt(x, 16));
|
||||
return `rgba(${r},${g},${b},${alpha})`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Makes sure that the document is loaded before executing wait()
|
||||
*/
|
||||
if (document.readyState !== 'loading') {
|
||||
wait();
|
||||
} else {
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
wait();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Add regions for all annotations on their corresponding wavesurfer instance
|
||||
*/
|
||||
function loadRegions() {
|
||||
for (var i = 0; i < numberAnnotations; i++) {
|
||||
var regions = window.prodigy.content.annotations[i];
|
||||
for (region in regions) {
|
||||
var color;
|
||||
var label = regions[region]['label'];
|
||||
if (!colorList.includes(label)) {
|
||||
colorList.push(label);
|
||||
}
|
||||
var id = colorList.indexOf(label) % prodigy.config.custom_theme.palettes.audio.length;
|
||||
var color = prodigy.config.custom_theme.palettes.audio[id];
|
||||
color = hex2rgba(color);
|
||||
var re = window['wavesurfer' + i].addRegion({ 'start': regions[region]['start'], 'end': regions[region]['end'], 'color': color, 'resize': false, 'drag': false, "attributes": { "label": label } });
|
||||
addRegionLabel(re, label);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create audio url
|
||||
* @see loadWave()
|
||||
* @see prodigyanswer
|
||||
*/
|
||||
async function createURL() {
|
||||
var blob;
|
||||
var track = document.querySelector('#track');
|
||||
var src = track.children[0].src
|
||||
var blob = await (await fetch(src)).blob()
|
||||
var objectURL = URL.createObjectURL(blob);
|
||||
|
||||
return objectURL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the same wavesurfer object as on Prodigy interface for all annotations
|
||||
* Those objects are muted
|
||||
*/
|
||||
async function loadWave() {
|
||||
var objectURL = await createURL();
|
||||
var wdict = {
|
||||
container: window.wavesurfer.container,
|
||||
audioRate: 1,
|
||||
autoCenter: true,
|
||||
autoCenterImmediately: false,
|
||||
backend: "WebAudio",
|
||||
barGap: 2,
|
||||
barHeight: 1,
|
||||
barMinHeight: null,
|
||||
barRadius: 2,
|
||||
barWidth: 0,
|
||||
cursorColor: "#333",
|
||||
cursorWidth: 1,
|
||||
fillParent: true,
|
||||
forceDecode: false,
|
||||
height: 128,
|
||||
hideScrollbar: false,
|
||||
interact: false,
|
||||
loopSelection: true,
|
||||
maxCanvasWidth: 4000,
|
||||
mediaControls: false,
|
||||
mediaType: "audio",
|
||||
normalize: false,
|
||||
partialRender: false,
|
||||
pixelRatio: 2,
|
||||
progressColor: "#583fcf",
|
||||
waveColor: "violet",
|
||||
removeMediaElementOnDestroy: true,
|
||||
responsive: false,
|
||||
scrollParent: true,
|
||||
skipLength: 2,
|
||||
splitChannels: false,
|
||||
plugins: [
|
||||
WaveSurfer.regions.create({})
|
||||
]
|
||||
};
|
||||
for (var i = 0; i < numberAnnotations; i++) {
|
||||
window['wavesurfer' + i] = WaveSurfer.create(wdict);
|
||||
window['wavesurfer' + i].load(objectURL);
|
||||
window['wavesurfer' + i].setMute(true);
|
||||
window['wavesurfer' + i].on('region-click', function (e) {
|
||||
var label = e.attributes.label;
|
||||
clickOnLabel(label);
|
||||
var re = window.wavesurfer.addRegion({ 'start': e.start, 'end': e.end, 'color': e.color });
|
||||
window.wavesurfer.fireEvent('region-update-end', re);
|
||||
});
|
||||
}
|
||||
var l = document.querySelector('wave').appendChild(document.createElement("span"));
|
||||
l.textContent = "Output";
|
||||
l.className = "title-wave";
|
||||
const nodeList = document.querySelectorAll('wave ~ wave');
|
||||
for (var i = 0; i < nodeList.length; i++) {
|
||||
nodeList[i].style.marginTop = "30px";
|
||||
nodeList[i].style.backgroundColor = "#0000000a";
|
||||
l = nodeList[i].appendChild(document.createElement("span"));
|
||||
l.textContent = "Input " + (i + 1);
|
||||
l.className = "title-wave";
|
||||
l.onclick = function () { test(i) };
|
||||
l.setAttribute("onclick", "addAllRegions(" + i + ");");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle click on annotation label
|
||||
* Add all regions of the clicked annotation on the main wavesurfer instance (the prodigy's one)
|
||||
* @param(i) indice of annotation (wavesurfer instance)
|
||||
*/
|
||||
function addAllRegions(i) {
|
||||
var rs = window['wavesurfer' + i].regions.list;
|
||||
num = Object.values(rs);
|
||||
for (var i in num) {
|
||||
var region = num[i];
|
||||
var label = region.attributes.label;
|
||||
clickOnLabel(label);
|
||||
var re = window.wavesurfer.addRegion({ 'start': region.start, 'end': region.end, 'color': region.color });
|
||||
window.wavesurfer.fireEvent('region-update-end', re);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle wavesurfer
|
||||
* Tracking audio playback for all annotations
|
||||
*/
|
||||
async function wait() {
|
||||
if (document.querySelector('#track') !== null) {
|
||||
await loadWave();
|
||||
loadRegions();
|
||||
window.wavesurfer.on('audioprocess', function (e) {
|
||||
var time = e / window.wavesurfer.getDuration();
|
||||
for (var i = 0; i < numberAnnotations; i++) {
|
||||
window['wavesurfer' + i].seekAndCenter(time);
|
||||
}
|
||||
});
|
||||
window.wavesurfer.on('seek', function (e) {
|
||||
for (var i = 0; i < numberAnnotations; i++) {
|
||||
window['wavesurfer' + i].seekTo(e);
|
||||
}
|
||||
});
|
||||
window.wavesurfer.on('zoom', function (e) {
|
||||
for (var i = 0; i < numberAnnotations; i++) {
|
||||
window['wavesurfer' + i].zoom(e);
|
||||
}
|
||||
});
|
||||
window.wavesurfer.on('finish', function (e) {
|
||||
for (var i = 0; i < numberAnnotations; i++) {
|
||||
window['wavesurfer' + i].seekTo(1);
|
||||
}
|
||||
});
|
||||
window.wavesurfer.on('pause', function (e) {
|
||||
for (var i = 0; i < numberAnnotations; i++) {
|
||||
window['wavesurfer' + i].pause();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
setTimeout(wait, 250);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load new audio and clear regions for all annotations wavesurfer when there is a new task
|
||||
*/
|
||||
document.addEventListener('prodigyanswer', async () => {
|
||||
var objectURL = await createURL();
|
||||
for (var i = 0; i < numberAnnotations; i++) {
|
||||
window['wavesurfer' + i].load(objectURL);
|
||||
window['wavesurfer' + i].clearRegions();
|
||||
}
|
||||
loadRegions();
|
||||
});
|
||||
|
||||
/**
|
||||
* Add CSS label to reference and hypothesis region
|
||||
* @see loadRegions()
|
||||
* @param(r) region object
|
||||
* @param(l) label
|
||||
*/
|
||||
function addRegionLabel(r, label) {
|
||||
var s = r.element
|
||||
var l = s.appendChild(document.createElement("span"))
|
||||
l.textContent = label,
|
||||
l.className = "pyannote-region",
|
||||
l.style.color = "rgb(0, 0, 0)",
|
||||
l.style.background = r.color,
|
||||
r.label = label;
|
||||
}
|
||||
@@ -1,220 +0,0 @@
|
||||
import base64
|
||||
from pathlib import Path
|
||||
from tempfile import mkstemp
|
||||
from typing import Any, Dict, Iterable, List
|
||||
|
||||
import prodigy
|
||||
from prodigy import set_hashes
|
||||
from prodigy.util import split_string
|
||||
from pyannote.core import Segment
|
||||
from pyannote.database import util
|
||||
|
||||
from pyannote.audio.core.io import Audio
|
||||
from pyannote.audio.pipelines import SpeakerDiarization
|
||||
|
||||
from ..common.utils import (
|
||||
AudioForProdigy,
|
||||
before_db,
|
||||
get_audio_spans,
|
||||
get_chunks,
|
||||
source_to_files,
|
||||
)
|
||||
|
||||
|
||||
def review_stream(
|
||||
source: Path,
|
||||
annotations: [dict],
|
||||
labels: [dict],
|
||||
diarization: bool = False,
|
||||
chunk: float = 30.0,
|
||||
) -> Iterable[Dict]:
|
||||
|
||||
files = source_to_files(source)
|
||||
audio_for_prodigy = AudioForProdigy()
|
||||
audio_for_pipeline = Audio(mono="downmix")
|
||||
chunks = get_chunks(source, chunk_duration=chunk)
|
||||
chunks = list(chunks)
|
||||
|
||||
files_annotations = {}
|
||||
for file in files:
|
||||
filename = file["text"]
|
||||
if diarization:
|
||||
list_annotations = [annotations[0][filename]] + [
|
||||
SpeakerDiarization.optimal_mapping(
|
||||
annotations[0][filename], ann[filename]
|
||||
)
|
||||
for ann in annotations[1:]
|
||||
]
|
||||
labels = [label for ann in list_annotations for label in ann.labels()]
|
||||
labels = list(dict.fromkeys(labels))
|
||||
files_annotations[filename] = (list_annotations, labels)
|
||||
else:
|
||||
list_annotations = [ann[filename] for ann in annotations]
|
||||
files_annotations[filename] = (list_annotations, labels)
|
||||
|
||||
for file, excerpt in chunks:
|
||||
path = file["path"]
|
||||
filename = file["text"]
|
||||
text = f"{filename} [{excerpt.start:.1f} - {excerpt.end:.1f}]"
|
||||
waveform, sample_rate = audio_for_pipeline.crop(path, excerpt, mode="pad")
|
||||
audio = audio_for_prodigy.crop(path, excerpt)
|
||||
duration = audio_for_pipeline.get_duration(path)
|
||||
list_spans = []
|
||||
for annotation in files_annotations[filename][0]:
|
||||
spans = get_audio_spans(annotation, excerpt, Segment(0, duration))
|
||||
list_spans.append(spans)
|
||||
|
||||
labels = files_annotations[filename][1]
|
||||
|
||||
yield {
|
||||
"path": path,
|
||||
"text": text,
|
||||
"audio": audio,
|
||||
"audio_spans": [],
|
||||
"annotations": list_spans,
|
||||
"config": {"labels": labels},
|
||||
"chunk": {"start": excerpt.start, "end": excerpt.end},
|
||||
"meta": {
|
||||
"file": filename,
|
||||
"start": f"{excerpt.start:.1f}",
|
||||
"end": f"{excerpt.end:.1f}",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@prodigy.recipe(
|
||||
"pyannote.review",
|
||||
dataset=("Dataset to save annotations to", "positional", None, str),
|
||||
source=(
|
||||
"Path to directory containing audio files whose annotation is to be checked",
|
||||
"positional",
|
||||
None,
|
||||
Path,
|
||||
),
|
||||
annotations=(
|
||||
"Comma-separated paths to annotation files",
|
||||
"positional",
|
||||
None,
|
||||
split_string,
|
||||
),
|
||||
chunk=(
|
||||
"Split long audio files into shorter chunks of that many seconds each",
|
||||
"option",
|
||||
None,
|
||||
float,
|
||||
),
|
||||
diarization=(
|
||||
"Optimal one-to-one mapping between the first annotation and the others",
|
||||
"flag",
|
||||
None,
|
||||
bool,
|
||||
),
|
||||
precision=("Cursor speed", "option", None, int),
|
||||
beep=("Beep when the player reaches the end of a region.", "flag", None, bool),
|
||||
)
|
||||
def review(
|
||||
dataset: str,
|
||||
source: Path,
|
||||
annotations: [List[str]],
|
||||
chunk: float = 30.0,
|
||||
diarization: bool = False,
|
||||
precision: int = 200,
|
||||
beep: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
|
||||
recipe_dir = Path(__file__).resolve().parent
|
||||
common_dir = recipe_dir.parent / "common"
|
||||
|
||||
controllerReview = recipe_dir / "controller.js"
|
||||
controller = common_dir / "controller.js"
|
||||
wavesurfer = common_dir / "wavesurfer.js"
|
||||
regions = common_dir / "regions.js"
|
||||
html = common_dir / "template.html"
|
||||
css = common_dir / "template.css"
|
||||
|
||||
template = common_dir / "instructions.html"
|
||||
png = common_dir / "commands.png"
|
||||
_, instructions_html = mkstemp(text=True)
|
||||
with open(controllerReview) as sc_review, open(wavesurfer) as s_wavesurfer, open(
|
||||
regions
|
||||
) as s_regions, open(html) as f_html, open(css) as f_css, open(
|
||||
controller
|
||||
) as sc, open(
|
||||
png, "rb"
|
||||
) as fp_png, open(
|
||||
instructions_html, "w"
|
||||
) as instructions_f, open(
|
||||
template, "r"
|
||||
) as fp_tpl:
|
||||
script_text = s_wavesurfer.read()
|
||||
script_text += "\n" + s_regions.read()
|
||||
script_text += "\n" + sc_review.read()
|
||||
script_text += "\n" + sc.read()
|
||||
templateH = f_html.read()
|
||||
templateC = f_css.read()
|
||||
b64 = base64.b64encode(fp_png.read()).decode("utf-8")
|
||||
instructions_f.write(fp_tpl.read().replace("{IMAGE}", b64))
|
||||
|
||||
list_annotations = [util.load_rttm(annotation) for annotation in annotations]
|
||||
|
||||
labels = [
|
||||
label
|
||||
for ann in list_annotations
|
||||
for anno in list(ann.values())
|
||||
for label in anno.labels()
|
||||
]
|
||||
|
||||
hashed_stream = (
|
||||
set_hashes(eg, input_keys=("path", "chunk"))
|
||||
for eg in review_stream(
|
||||
source, list_annotations, labels, diarization=diarization, chunk=chunk
|
||||
)
|
||||
)
|
||||
|
||||
return {
|
||||
"view_id": "blocks",
|
||||
"dataset": dataset,
|
||||
"stream": hashed_stream,
|
||||
"before_db": before_db,
|
||||
"config": {
|
||||
"global_css": templateC,
|
||||
"javascript": script_text,
|
||||
"instructions": instructions_html,
|
||||
"precision": precision,
|
||||
"beep": beep,
|
||||
"show_audio_minimap": False,
|
||||
"audio_bar_width": 0,
|
||||
"audio_bar_height": 1,
|
||||
"number_annotations": len(annotations),
|
||||
"labels": labels,
|
||||
"custom_theme": {
|
||||
"palettes": {
|
||||
"audio": [
|
||||
"#ffd700",
|
||||
"#00ffff",
|
||||
"#ff00ff",
|
||||
"#00ff00",
|
||||
"#9932cc",
|
||||
"#00bfff",
|
||||
"#ff7f50",
|
||||
"#66cdaa",
|
||||
],
|
||||
}
|
||||
},
|
||||
"blocks": [
|
||||
{
|
||||
"view_id": "audio_manual",
|
||||
},
|
||||
{"view_id": "html", "html_template": templateH},
|
||||
],
|
||||
"show_audio_timeline": True,
|
||||
"buttons": ["accept", "ignore", "undo"],
|
||||
"keymap": {
|
||||
"accept": ["enter"],
|
||||
"ignore": ["escape"],
|
||||
"undo": ["u"],
|
||||
"playpause": ["space"],
|
||||
},
|
||||
"show_flag": True,
|
||||
},
|
||||
}
|
||||
@@ -48,8 +48,6 @@ dev =
|
||||
pre_commit>=2.7
|
||||
recommonmark>=0.6
|
||||
black>=22.3.0
|
||||
interactive =
|
||||
prodigy>=1.11.0
|
||||
cli =
|
||||
hydra-core >=1.1,<1.2
|
||||
typer >= 0.4.0,<0.5.0
|
||||
@@ -60,10 +58,6 @@ console_scripts =
|
||||
pyannote-audio-train=pyannote.audio.cli.train:train
|
||||
pyannote-audio-eval=pyannote.audio.cli.evaluate:evaluate
|
||||
|
||||
prodigy_recipes =
|
||||
pyannote.audio = pyannote.audio.interactive.pipeline.recipe:pipeline
|
||||
pyannote.review = pyannote.audio.interactive.review.recipe:review
|
||||
pyannote.diff = pyannote.audio.interactive.diff.recipe:diff
|
||||
|
||||
[test]
|
||||
# py.test options when running `python setup.py test`
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
# Annotating your own data with 💥 Prodigy
|
||||
|
||||
Manually segmenting and labeling audio data is time consuming. For speaker diarization, depending on the required level of precision, it may take more than 10 times the duration of a recording to annotate it.
|
||||
|
||||
## Table of content
|
||||
|
||||
* [Recipes](#recipes)
|
||||
* [Keyboard shortcuts](#keyboard-shortcuts)
|
||||
|
||||
## Recipes
|
||||
|
||||
`pyannote.audio` comes with a bunch of [💥 Prodigy](https://prodi.gy) recipes designed to speed things up a bit.
|
||||
|
||||
| Recipe | Usage |
|
||||
|-----------------------|-------------------------------------------------------|
|
||||
🦻 `pyannote.audio` | Annotate with a [pretrained pipeline](https://huggingface.co/models?other=pyannote-audio-pipeline) in the loop
|
||||
🧐 `pyannote.review` | Merge multiple annotations
|
||||
🤲 `pyannote.diff` | Show differences between two annotations
|
||||
🗄 `pyannote.database` | Dump annotations as [`pyannote.database`](https://github.com/pyannote/pyannote-database/) protocols
|
||||
|
||||
### 🦻 `pyannote.audio` | Annotate with a pretrained pipeline in the loop
|
||||
|
||||
```bash
|
||||
prodigy pyannote.audio dataset /path/to/audio/directory pyannote/speaker-segmentation
|
||||
```
|
||||
|
||||

|
||||
|
||||
`pyannote.audio` recipe will stream in `.wav` files in chunks and apply [a pretrained pipeline](https://huggingface.co/models?other=pyannote-audio-pipeline). You can then adjust the regions manually if needed.
|
||||
|
||||
|
||||
<details>
|
||||
<summary>More options</summary>
|
||||
|
||||
```
|
||||
prodigy pyannote.audio [options] dataset source pipeline
|
||||
|
||||
dataset Prodigy dataset to save annotations to.
|
||||
source Path to directory containing audio files to annotate.
|
||||
pipeline Name of pretrained pipeline on huggingface.co (e.g.
|
||||
pyannote/speaker-segmentation) or path to local YAML file.
|
||||
-chunk DURATION Split audio files into shorter chunks of that many seconds.
|
||||
Defaults to 10s.
|
||||
-precision STEP Temporal precision of keyboard controls, in milliseconds.
|
||||
Defaults to 200ms.
|
||||
-beep Produce a beep when the player reaches the end of a region.
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
### 🧐 `pyannote.review` | Merge multiple annotations
|
||||
|
||||
```bash
|
||||
prodigy pyannote.review dataset /path/to/audio/directory input1.rttm,input2.rttm
|
||||
```
|
||||
|
||||

|
||||
|
||||
`pyannote.review` recipe take as many annotation files, using the RTTM file format, as you want and let you compare and choose which ones are best within the same stream as `pyannote.audio` recipe.
|
||||
Click on a segment of the annotation files to add it to the ouput audio, or on "Input X" to add all segments at once.
|
||||
|
||||
<details>
|
||||
<summary>More options</summary>
|
||||
|
||||
```
|
||||
prodigy pyannote.review [options] dataset source annotations
|
||||
|
||||
dataset Prodigy dataset to save annotations to.
|
||||
source Path to directory containing audio files whose annotation is to be checked.
|
||||
annotations Comma-separated paths to annotation files.
|
||||
-chunk DURATION Split audio files into shorter chunks of that many seconds.
|
||||
Defaults to 30s.
|
||||
-diarization Make a optimal one-to-one mapping between the first annotation and the others.
|
||||
-precision STEP Temporal precision of keyboard controls, in milliseconds.
|
||||
Defaults to 200ms.
|
||||
-beep Produce a beep when the player reaches the end of a region.
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
### 🤲 `pyannote.diff` | Show differences between two annotations
|
||||
|
||||
```bash
|
||||
prodigy pyannote.diff dataset /path/to/audio/directory /path/to/reference.rttm /path/to/hypothesis.rttm
|
||||
```
|
||||
|
||||

|
||||
|
||||
`pyannote.diff` recipe take one reference file and one hypothesis file, using the RTTM file format, and focus where there are the most errors among missed detections, false alarms and confusions.
|
||||
You can filter on one or more error types and their minimum duration with the corresponding options.
|
||||
|
||||
|
||||
<details>
|
||||
<summary>More options</summary>
|
||||
|
||||
```
|
||||
prodigy pyannote.diff [options] dataset source reference hypothesis
|
||||
|
||||
dataset Prodigy dataset to save annotations to.
|
||||
source Path to directory containing audio files whose annotation is to be checked.
|
||||
reference Path to reference file.
|
||||
hypothesis Path to hypothesis file.
|
||||
-chunk DURATION Split audio files into shorter chunks of that many seconds.
|
||||
Defaults to 30s.
|
||||
-min-duration DURATION Minimum duration of errors in ms.
|
||||
Defaults to 200ms.
|
||||
-diarization Make a optimal one-to-one mapping between reference and hypothesis.
|
||||
-false-alarm Display false alarm errors.
|
||||
-speaker-confusion Display confusion errors.
|
||||
-missed-detection Display missed detection errors.
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### 🗄 `pyannote.database` | Dump annotations as `pyannote.database` protocols
|
||||
|
||||
Work in progress
|
||||
|
||||
|
||||
## Keyboard shortcuts
|
||||
|
||||
Though `pyannote.audio` recipes are built on top of the Prodigy [audio interface](https://prodi.gy/docs/api-interfaces#audio), they provide a bunch of handy additional keyboard shortcuts.
|
||||
|
||||
| Shortcut | Description |
|
||||
|-----------------------------------|--------------------------------------------------|
|
||||
| `left` / `right` (+ `w`) | Shift player cursor (speed up) |
|
||||
| `up` / `down` | Switch active region |
|
||||
| `shift + left` / `shift + right` | Shift active region start time |
|
||||
| `ctrl + left` / `ctrl + right` | Shift active region end time |
|
||||
| `shift + up` | Create a new region |
|
||||
| `shift + down` / `backspace` | Remove active region |
|
||||
| `spacebar` | Play/pause player |
|
||||
| `escape` | Ignore this sample |
|
||||
| `enter` | Validate annotation |
|
||||
|
||||
|
||||
## RTTM file format
|
||||
|
||||
RTTM files contain one line per speech turn, using the following convention:
|
||||
|
||||
```bash
|
||||
SPEAKER {uri} 1 {start_time} {duration} <NA> <NA> {speaker_id} <NA> <NA>
|
||||
```
|
||||
* uri: file identifier (as given by pyannote.database protocols)
|
||||
* start_time: speech turn start time in seconds
|
||||
* duration: speech turn duration in seconds
|
||||
* confidence: confidence score (can be anything, not used for now)
|
||||
* gender: speaker gender (can be anything, not used for now)
|
||||
* speaker_id: speaker identifier
|
||||
Reference in new issue
Block a user