chore: remove prodigy support (#1469)

This commit is contained in:
Hervé BREDIN authored and GitHub committed 2023-09-26 10:09:28 +02:00
1 parent d169fd3770
commit 76d86fa97a
17 files changed
-9883

No files matched your search

-1
View File
@@ -74,7 +74,6 @@ pip install -qq https://github.com/pyannote/pyannote-audio/archive/refs/heads/de
- 2021-08-05 > ["Streaming voice activity detection with pyannote.audio"](https://herve.niderb.fr/fastpages/2021/08/05/Streaming-voice-activity-detection-with-pyannote.html)
- Miscellaneous
- [Training with `pyannote-audio-train` command line tool](tutorials/training_with_cli.md)
- [Annotating your own data with Prodigy](tutorials/prodigy.md)
- [Speaker verification](tutorials/speaker_verification.ipynb)
- Visualization and debugging
Binary file not shown.

Before

Width:  |  Height:  |  Size: 135 KiB

@@ -1,367 +0,0 @@
// The MIT License (MIT)
//
// Copyright (c) 2021 CNRS
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
// Can't create constant because prodigy reload this file every batch
var currentRegion = 0;
var regions = null;
var ids = null;
var refresh = true;
var left = 'ArrowLeft';
var right = 'ArrowRight';
var startR = 'Shift';
var endR = 'Control';
var PRECISION = (prodigy.config.precision / 1000);
var BEEP = prodigy.config.beep;
var EXCERPT = 1;
var keysMap = {};
/**
* Handle web audio for beep
* @see beep()
*/
var audioCtx = new(window.AudioContext || window.webkitAudioContext)();
/**
* Makes sure that the document is loaded before executing waitForElement()
* Add a small timeout if wavesurfer is already defined (useful when a new batch is coming),
* the time for prodigy to update wavesurfer.
* @see waitForElement()
*/
if(document.readyState !== 'loading') {
if(typeof window.wavesurfer !== "undefined"){
setTimeout(waitForElement,25);
}else{
waitForElement();
}
} else {
document.addEventListener('DOMContentLoaded', function () {
waitForElement();
});
}
/**
* Compare if a region is before or after another one
* Useful to sort global variable 'ids'
* @see reloadWave()
* @param {region1} Region object
* @param {region2} Region Object
* @return {number} -1 if region1 start before, 1 otherwise, 0 if the identical
*/
function compare(region1, region2){
if(region1.start < region2.start){
return -1;
}else if (region1.start > region2.start){
return 1;
}else{
return 0;
}
}
/**
* Simulate a click on prodigy's label radio button
* Only useful for "review recipe"
* Note: might break with future versions of Prodigy
* @param {label} Label to click on
*/
function clickOnLabel(label){
document.querySelector("input[type=radio][value=\'"+label+"\']").click()
}
/**
* Update prodigy.content with all regions from window.wavesurfer.regions.list (thus, all regions that can be seen in the interface)
* Discuss in this issue :
* https://support.prodi.gy/t/weird-interaction-between-window-prodigy-update-and-wavesurfer/5450
*/
function updateContent(){
var regions = window.wavesurfer.regions.list;
var content = [];
for (var id in regions){
var region = regions[id];
content.push({start : region.start, end : region.end, label : region.label, id : region.id, color : region.color});
}
window.prodigy.update({audio_spans : content});
}
/**
* Create a beep sound from scratch
* You can adjust the gain, frequency (here its A 440) and duration
*/
function beep() {
if(BEEP){
var oscillator = audioCtx.createOscillator();
var gainNode = audioCtx.createGain();
oscillator.connect(gainNode);
gainNode.connect(audioCtx.destination);
gainNode.gain.value = 0.1;
oscillator.frequency.value = 440;
oscillator.type = "square";
oscillator.start();
setTimeout(
function() {
oscillator.stop();
},
150 // FIXME: should depend on the value of "precision"
);
}
}
/**
* Change CSS style for the selected region
* @param {e} Region object
*/
function activeRegion(e){
e.element.style.borderTop = "3px solid";
e.element.style.borderBottom = "3px solid";
}
/**
* @see activeRegion()
* Undo CSS change
* @param {e} Region object
*/
function deactiveRegion(e){
e.element.style.borderTop = "";
e.element.style.borderBottom = "";
}
/**
* Update global variables 'regions' and 'ids' with regions in window.wavesurfer.regions.list
* Put the first one as "active" and update the variable currentRegion
*/
function reloadWave(){
regions = window.wavesurfer.regions.list;
ids = Object.values(regions);
ids.sort(compare);
if(ids.length > 0){
currentRegion = 0;
activeRegion(ids[0]);
}
}
/**
* Switch selected region
* Update var currentRegion
* Place wavesurfer cursor at the beginning of the new region or the beginning of the file if it's a new prodigy task
* @see activeRegion() / deactiveRegion()
* @param {ids} Ids of the region to be selected
*/
function switchCurrent(newId){
if(ids.length > 0){
deactiveRegion(ids[currentRegion]);
currentRegion = newId;
activeRegion(ids[newId])
if(refresh){
window.wavesurfer.seekTo(0);
}else{
var time = (ids[currentRegion].start) / (window.wavesurfer.getDuration());
window.wavesurfer.seekTo(time);
}
}
}
/**
* Handle wavesurfer regions
* Add event listener to some wavesurfer event
*/
function waitForElement(){
if(typeof window.wavesurfer !== "undefined"){
reloadWave();
// Select created region or the first one if it's a new task
window.wavesurfer.on('region-created', function(e){
setTimeout(function(){
if(ids.length > 0) deactiveRegion(ids[currentRegion]);
reloadWave();
if(refresh){
switchCurrent(0);
}else{
switchCurrent(ids.indexOf(e));
}
}, 5);
});
// Change region label (by remove the old one and create a new one with proper label)
window.wavesurfer.on('region-dblclick',function(e){
re = window.wavesurfer.addRegion({'start' : e.start,'end' : e.end});
e.remove();
window.wavesurfer.fireEvent('region-update-end',re);
});
// Select region on click
window.wavesurfer.on('region-click',function(e){
switchCurrent(ids.indexOf(e));
});
// Beep when region end
window.wavesurfer.on('region-out',function(e){
beep();
});
// @see updateContent()
window.wavesurfer.on('region-update-end', function(e){
updateContent();
});
// @see updateContent()
// Switch selected region when deleted
window.wavesurfer.on('region-removed',function(e){
updateContent();
if(currentRegion == (ids.length - 1)){
var newId = 0;
}else{
var newId = currentRegion;
}
reloadWave();
if(ids.length > 0) switchCurrent(newId);
});
}else{
setTimeout(waitForElement, 250);
}
}
// Check if it's a new prodigy task
document.addEventListener('prodigyanswer', e => {
refresh = true;
})
/**
* Keyboard controller
* | Key 1 | Key 2 | Command |
* | ------------- | ------------- | ------------ |
* | Arrows left/right | [W] | Move Cursor [speed up] |
* | Shift | Arrows left/right | Change start of current segment |
* | Control | Arrows left/right | Change end of current segment |
* | Arrows up/down | | Change current segment to the next/precedent one |
* | Shift | Arrows up/[down] | Create [or remove] segment |
* | Backspace | | Remove current segment |
*/
document.querySelector('#root').onkeydown = document.querySelector('#root').onkeyup = function(e){
e = e || event;
keysMap[e.key] = e.type == 'keydown';
var pos = window.wavesurfer.getCurrentTime();
var audioEnd = window.wavesurfer.getDuration();
var region = ids[currentRegion];
refresh = false;
// If Left is pressed
if(keysMap[left] && !keysMap[right]){
// If Shift is pressed
if(keysMap[startR] && !keysMap[endR]){
// Shortens start if possible
if((region.start - PRECISION) <= 0){
region.update({'start' : 0});
window.wavesurfer.fireEvent('region-update-end',region);
window.wavesurfer.play(0, region.end);
}else{
region.update({'start' : region.start - PRECISION });
window.wavesurfer.fireEvent('region-update-end',region);
window.wavesurfer.play(region.start, region.end);
}
// If Ctrl is pressed
}else if(keysMap[endR] && !keysMap[startR]){
var startTime = region.end - EXCERPT;
if(startTime < region.start) startTime = region.start;
// Shortens end if possible
if((region.end - PRECISION) > region.start){
region.update({'end' : region.end - PRECISION });
window.wavesurfer.fireEvent('region-update-end',region);
window.wavesurfer.play(startTime, region.end);
}
}else{
// Else change cursor position
// Speed up naviguation if W is pressed
if(keysMap['w']){
var time = (pos - PRECISION*2) / audioEnd;
}else{
var time = (pos - PRECISION) / audioEnd;
}
if(time < 0) time = 0;
window.wavesurfer.pause();
window.wavesurfer.seekTo(time);
}
// If Right is pressed
}else if(keysMap[right] && !keysMap[left]){
// If Shift is pressed
if(keysMap[startR] && !keysMap[endR]){
// Extend start if possible
if(region.start + PRECISION < region.end){
region.update({'start' : region.start + PRECISION });
window.wavesurfer.fireEvent('region-update-end',region);
window.wavesurfer.play(region.start, region.end);
}
// If Ctrl is pressed
}else if(keysMap[endR] && !keysMap[startR]){
// Extend end if possible (while keep playing the audio)
if(!window.wavesurfer.isPlaying()){
var startTime = region.end - EXCERPT;
if(startTime < region.start) startTime = region.start;
}else{
var startTime = pos;
}
if((region.end + PRECISION) >= audioEnd){
region.update({'end' : audioEnd });
window.wavesurfer.fireEvent('region-update-end',region);
}else{
region.update({'end' : region.end + PRECISION });
window.wavesurfer.fireEvent('region-update-end',region);
}
window.wavesurfer.play(startTime, region.end);
}else{
// Else change cursor position
// Speed up naviguation if W is pressed
if(keysMap['w']){
var time = (pos + PRECISION*2) / audioEnd;
}else{
var time = (pos + PRECISION) / audioEnd;
}
if(time > 1) time = 1;
window.wavesurfer.pause();
window.wavesurfer.seekTo(time);
}
// If Up and shift is pressed : new region
}else if (keysMap['ArrowUp'] && keysMap['Shift']){
var fin = pos + 1;
if(fin > audioEnd) fin = audioEnd;
re = window.wavesurfer.addRegion({'start' : pos,'end' : fin});
window.wavesurfer.fireEvent('region-update-end',re);
// If Down and Shift or Backspace: delete region
// Check backspace for diarization text field
}else if(keysMap['Backspace'] || (keysMap['ArrowDown'] && keysMap['Shift'])){
ids[currentRegion].remove();
// If Up/Down @see switchCurrent
}else if(keysMap['ArrowUp']){
if(currentRegion == (ids.length - 1)){
switchCurrent(0);
}else{
switchCurrent(currentRegion + 1);
}
}else if(keysMap['ArrowDown']){
if(currentRegion == 0){
switchCurrent(ids.length - 1);
}else{
switchCurrent(currentRegion - 1);
}
}else if(keysMap['u']){
reloadWave();
}
}
@@ -1,69 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<h1>Commands</h1>
</head>
<body>
You have to mark the speech moments with bounding boxes.
<br/>
<img src='data:image/png;base64,{IMAGE}' with="812" height="284">
<br/><br/>
<table>
<tr>
<th>Key 1</th>
<th>Key 2</th>
<th>Command</th>
</tr>
<tr>
<td>Arrows left/right</td>
<td>[W]</td>
<td>Move Cursor [speed up]</td>
</tr>
<tr>
<td>Shift</td>
<td>Arrow left/right</td>
<td>Change start of current segment</td>
</tr>
<tr>
<td>Control</td>
<td>Arrow left/right</td>
<td>Change end of current segment</td>
</tr>
<tr>
<td>Arrows up/down</td>
<td></td>
<td>Change current segment to the next/precedent one</td>
</tr>
<tr>
<td>Shift</td>
<td>Arrow up/[down]</td>
<td>Create or [remove] segment</td>
</tr>
<tr>
<td>Backspace</td>
<td></td>
<td>Remove current segment</td>
</tr>
<tr>
<td>Spacebar</td>
<td></td>
<td>Play/pause audio</td>
</tr>
<tr>
<td>U</td>
<td></td>
<td>Undo all actions</td>
</tr>
<tr>
<td>Escape</td>
<td></td>
<td>Ignore this sample</td>
</tr>
<tr>
<td>Enter</td>
<td></td>
<td>Validate annotation</td>
</tr>
</table>
</body>
</html>
File diff suppressed because it is too large. Load diff
@@ -1,41 +0,0 @@
.pyannote-region {
left: 1px;
color: #000;
bottom: 0;
display: block;
padding: 0 3px;
z-index: 10;
position: absolute;
font-size: 0.7em;
background: #ffe184;
font-weight: bold;
font-family: "Roboto Condensed", "Arial Narrow", sans-serif;
user-select: none;
text-transform: uppercase;
}
.title-wave {
left: 1px;
color: #000;
display: block;
padding: 0 3px;
z-index: 10;
position: absolute;
font-size: 0.7em;
background: #ddd;
font-weight: bold;
font-family: "Lato", "Trebuchet MS", Roboto, Helvetica, Arial, sans-serif;
user-select: none;
text-transform: uppercase;
}
.prodigy-content{ white-space: inherit }
wave{ overflow: hidden !important;}
.legend { list-style: none; }
.legend li { float: left; margin-right: 10px; }
.legend span { border: 1px solid #ccc; float: left; width: 30px; height: 30px; margin: 2px; }
.legend .falsealarm { background-color: #9932cc33; }
.legend .confusion { background-color: #ff634733; }
.legend .misseddetection { background-color: #00ffff33; }
@@ -1,3 +0,0 @@
<audio id='track'>
<source src={{audio}} type='audio/x-wav'/>
</audio>
-169
View File
@@ -1,169 +0,0 @@
# The MIT License (MIT)
#
# Copyright (c) 2021 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import base64
import io
from pathlib import Path
from typing import Dict, List, Optional, Text
import numpy as np
import scipy.io.wavfile
from prodigy.components.loaders import Audio as ProdigyAudioLoader
from pyannote.core import Annotation, Segment, SlidingWindow
from pyannote.audio import Audio
class AudioForProdigy(Audio):
def __init__(self):
super().__init__(sample_rate=16000, mono="downmix")
def crop(self, path: Path, excerpt: Segment) -> Text:
waveform, _ = super().crop(path, excerpt)
waveform = waveform.numpy().T
waveform /= np.max(np.abs(waveform)) + 1e-8
with io.BytesIO() as content:
scipy.io.wavfile.write(content, self.sample_rate, waveform)
content.seek(0)
b64 = base64.b64encode(content.read()).decode()
b64 = f"data:audio/x-wav;base64,{b64}"
return b64
def to_audio_spans(annotation: Annotation, focus: Segment = None) -> Dict:
"""Convert pyannote.core.Annotation to Prodigy's audio_spans
Parameters
----------
annotation : Annotation
Annotation with t=0s time origin.
focus : Segment, optional
When provided, use its start time as audio_spans time origin.
Returns
-------
audio_spans : list of dict
"""
shift = 0.0 if focus is None else focus.start
return [
{"start": segment.start - shift, "end": segment.end - shift, "label": label}
for segment, _, label in annotation.itertracks(yield_label=True)
]
def get_audio_spans(
annotation: Annotation, excerpt: Segment, excerpt_with_context: Segment = None
):
excerpt_with_context = excerpt_with_context or excerpt
shift = excerpt.start - excerpt_with_context.start
shifted_excerpt = Segment(start=shift, end=shift + excerpt.duration)
excerpt_annotation = annotation.crop(shifted_excerpt)
return [
{"start": segment.start - shift, "end": segment.end - shift, "label": label}
for segment, _, label in excerpt_annotation.itertracks(yield_label=True)
]
def remove_audio_before_db(examples: List[Dict]) -> List[Dict]:
"""Remove (potentially heavy) 'audio' key from examples
Parameters
----------
examples : list of dict
Examples.
Returns
-------
examples : list of dict
Examples with 'audio' key removed.
"""
for eg in examples:
if "audio" in eg:
del eg["audio"]
return examples
def source_to_files(source: Path) -> List[Dict]:
"""
Convert a directory or a file path to a list of files object for prodigy
"""
if source.is_dir():
files = ProdigyAudioLoader(source)
else:
name = source.stem
files = [{"path": source, "text": name, "meta": {"file": source}}]
return files
def get_chunks(source: Path, chunk_duration: Optional[float] = None):
files = source_to_files(source)
audio = Audio(mono="downmix")
for file in files:
duration = audio.get_duration(file["path"])
if (chunk_duration is None) or (duration < chunk_duration):
chunks = [Segment(0, duration)]
else:
windows = SlidingWindow(
start=0.0, duration=chunk_duration, step=chunk_duration
)
chunks = windows(Segment(0, duration), align_last=True)
for chunk in chunks:
yield file, chunk
def before_db(examples):
"""Post-process examples before sending them to the database
1. Remove "audio" key as it is very heavy and can easily be retrieved from other keys
2. Shift Prodigy/wavesurfer chunk-based audio spans so that their timing are file-based.
"""
for eg in examples:
# 1. remove "audio" key as it can be retrieved from "chunk" and "path" keys
if "audio" in eg:
del eg["audio"]
# 2. shift audio spans
chunk_start = eg["chunk"]["start"]
audio_spans_keys = [key for key in eg if "audio_spans" in key]
for key in audio_spans_keys:
eg[key] = [
{
"start": span["start"] + chunk_start,
"end": span["end"] + chunk_start,
"label": span["label"],
}
for span in eg[key]
]
return examples
File diff suppressed because it is too large. Load diff
@@ -1,231 +0,0 @@
// The MIT License (MIT)
//
// Copyright (c) 2021 CNRS
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
var numberAnnotations = 2;
var colorList = [];
var hex2rgba = (hex, alpha = 0.2) => {
const [r, g, b] = hex.match(/\w\w/g).map(x => parseInt(x, 16));
return `rgba(${r},${g},${b},${alpha})`;
};
/**
* Makes sure that the document is loaded before executing waitForElement()
* @see waitForElement()
*/
if(document.readyState !== 'loading') {
waitForElement();
} else {
document.addEventListener('DOMContentLoaded', function () {
waitForElement();
});
}
/**
* Remove label on region (created by prodigy)
* @param(r) region object
* @see waitForElement()
*/
function removeLabel(r){
s = r.start;
e = r.end;
c = r.color;
r.remove();
window.wavesurfer.addRegion({"start" : s, "end" :e, 'color' : c, "resize": false, "drag": false});
}
/**
* Add reference and hypothesis regions on the corresponding wavesurfer instance
*/
function loadRegions(){
for(var i=0; i < numberAnnotations;i++){
(i == 0)? (regions = window.prodigy.content.reference) : (regions = window.prodigy.content.hypothesis)
for (region in regions){
var label = regions[region]['label'];
if (!(label in colorList)){
colorList.push(label);
}
var id = colorList.indexOf(label) % prodigy.config.custom_theme.palettes.audio.length;
var color = prodigy.config.custom_theme.palettes.audio[id];
color = hex2rgba(color);
var re = window['wavesurfer'+i].addRegion({'start' : regions[region]['start'],'end' : regions[region]['end'],'color' : color, 'resize' : false, 'drag' : false, "attributes": {"label":regions[region]['label']}});
addRegionLabel(re,label);
}
}
}
/**
* Create audio url for reference and hypothesis wavesurfer instance
* @see loadWave()
* @see LoadTrack()
*/
async function createURL(){
var blob;
var track = document.querySelector('#track');
var src = track.children[0].src
var blob = await (await fetch(src)).blob()
var objectURL = URL.createObjectURL(blob);
return objectURL;
}
/**
* Create the same wavesurfer object as on Prodigy interface for reference and hypothesis
* Those objects are muted
*/
async function loadWave(){
var objectURL = await createURL();
var wdict = {
container: window.wavesurfer.container,
audioRate: 1,
autoCenter: true,
autoCenterImmediately: false,
backend: "WebAudio",
barGap: 2,
barHeight: 1,
barMinHeight: null,
barRadius: 2,
barWidth: 0,
cursorColor: "#333",
cursorWidth: 1,
fillParent: true,
forceDecode: false,
height: 128,
hideScrollbar: false,
interact: false,
loopSelection: true,
maxCanvasWidth: 4000,
mediaControls: false,
mediaType: "audio",
normalize: false,
partialRender: false,
pixelRatio: 2,
progressColor: "#583fcf",
waveColor: "violet",
removeMediaElementOnDestroy: true,
responsive: false,
scrollParent: true,
skipLength: 2,
splitChannels: false,
plugins: [
WaveSurfer.regions.create({})
]
};
for(var i=0; i < numberAnnotations;i++){
window['wavesurfer'+i] = WaveSurfer.create(wdict);
window['wavesurfer'+i].load(objectURL);
window['wavesurfer'+i].setMute(true);
}
var l = document.querySelector('wave').appendChild(document.createElement("span"));
l.textContent= "Errors";
l.className = "title-wave";
const nodeList = document.querySelectorAll('wave ~ wave');
for(var i = 0; i < nodeList.length;i++){
nodeList[i].style.marginTop = "30px";
nodeList[i].style.backgroundColor = "#0000000a";
l = nodeList[i].appendChild(document.createElement("span"));
(i == 0)? (l.textContent = "Reference") : (l.textContent = "Hypothesis")
l.className = "title-wave";
}
}
/**
* Handle wavesurfer
* Tracking audio playback for reference and hypothesis wavesurfer
*/
async function waitForElement(){
if((typeof window.wavesurfer !== "undefined") && (document.querySelector('#track') !== null)){
window.wavesurfer.on('region-created', function(e){
setTimeout(function(){
if("label" in e){
removeLabel(e);
}},5);
})
window.wavesurfer.on('audioprocess', function(e){
var time = e / window.wavesurfer.getDuration();
for(var i=0; i < numberAnnotations;i++){
window['wavesurfer'+i].seekAndCenter(time);
}
});
window.wavesurfer.on('seek', function(e){
for(var i=0; i < numberAnnotations;i++){
window['wavesurfer'+i].seekTo(e);
}
});
window.wavesurfer.on('zoom', function(e){
for(var i=0; i < numberAnnotations;i++){
window['wavesurfer'+i].zoom(e);
}
});
window.wavesurfer.on('finish', function(e){
for(var i=0; i < numberAnnotations;i++){
window['wavesurfer'+i].seekTo(1);
}
});
window.wavesurfer.on('pause', function(e){
for(var i=0; i < numberAnnotations;i++){
window['wavesurfer'+i].pause();
}
});
await loadWave();
loadRegions();
}else{
setTimeout(waitForElement, 250);
}
}
/**
* Reload audio (when there is a new task)
* @see prodigyanswer
*/
async function loadTrack(){
if(document.querySelector('#track') !== null){
var objectURL = await createURL();
for(var i=0; i < numberAnnotations;i++){
window['wavesurfer'+i].load(objectURL);
window['wavesurfer'+i].clearRegions();
}
loadRegions();
}else{
setTimeout(loadTrack, 250);
}
}
// Check if it's a new prodigy task
document.addEventListener('prodigyanswer', async() => {
loadTrack();
});
/**
* Add CSS label to reference and hypothesis region
* @see loadRegions()
* @param(r) region object
* @param(l) label
*/
function addRegionLabel(r,label){
var s = r.element
var l = s.appendChild(document.createElement("span"))
l.textContent = label,
l.className = "pyannote-region",
l.style.color = "rgb(0, 0, 0)",
l.style.background = r.color,
r.label = label;
}
@@ -1,5 +0,0 @@
<ul class="legend">
<li><span class="confusion"></span>Confusion</li>
<li><span class="falsealarm"></span> False Alarm</li>
<li><span class="misseddetection"></span>Missed Detection</li>
</ul>
-258
View File
@@ -1,258 +0,0 @@
from pathlib import Path
from typing import Any, Dict, Iterable, List
import prodigy
from prodigy import set_hashes
from pyannote.core import Annotation, Segment
from pyannote.database import util
from pyannote.metrics.errors.identification import IdentificationErrorAnalysis
from pyannote.audio import Audio
from pyannote.audio.pipelines import SpeakerDiarization
from ..common.utils import (
AudioForProdigy,
before_db,
get_audio_spans,
get_chunks,
source_to_files,
)
def diff_stream(
source: Path,
reference: dict,
hypothesis: dict,
list_errors: List,
diarization: bool = False,
chunk: float = 30.0,
min_duration: int = 200,
) -> Iterable[Dict]:
files = source_to_files(source)
files_errors = {}
for file in files:
filename = file["text"]
ref = reference[filename]
hyp = hypothesis[filename]
if diarization:
hyp: Annotation = SpeakerDiarization.optimal_mapping(ref, hyp)
identificationErrorAnalysis = IdentificationErrorAnalysis()
errors = identificationErrorAnalysis.difference(ref, hyp)
newLabels = {}
for labels in errors.labels():
a, b, c = labels
newLabels[(a, b, c)] = a
errors = errors.rename_labels(newLabels)
errors = errors.subset(["correct"], invert=True)
if list_errors[0] or list_errors[1] or list_errors[2]:
if not list_errors[0]:
errors = errors.subset(["false alarm"], invert=True)
if not list_errors[1]:
errors = errors.subset(["confusion"], invert=True)
if not list_errors[2]:
errors = errors.subset(["missed detection"], invert=True)
clean_errors = Annotation()
for segment, track, label in errors.itertracks(yield_label=True):
if segment.duration * 1000 > min_duration:
clean_errors[segment, track] = label
errors = clean_errors
files_errors[filename] = errors
audio_for_prodigy = AudioForProdigy()
audio_for_pipeline = Audio(mono="downmix")
chunks = get_chunks(source, chunk_duration=chunk)
chunks = list(chunks)
chunks = sorted(
chunks,
key=lambda k: max(
(
files_errors[k[0]["text"]]
.crop(k[1], mode="intersection")
.label_duration(e)
for e in files_errors[k[0]["text"]]
.crop(k[1], mode="intersection")
.labels()
),
default=0,
),
reverse=True,
)
for file, excerpt in chunks:
path = file["path"]
filename = file["text"]
text = f"{filename} [{excerpt.start:.1f} - {excerpt.end:.1f}]"
duration = audio_for_pipeline.get_duration(path)
# load audio excerpt
waveform, sample_rate = audio_for_pipeline.crop(path, excerpt, mode="pad")
# load audio excerpt for visualization in Prodigy
audio = audio_for_prodigy.crop(path, excerpt)
audio_spans = get_audio_spans(
files_errors[filename], excerpt, Segment(0, duration)
)
ref = get_audio_spans(reference[filename], excerpt, Segment(0, duration))
hyp = get_audio_spans(hypothesis[filename], excerpt, Segment(0, duration))
yield {
"path": path,
"text": text,
"audio": audio,
"audio_spans": audio_spans,
"reference": ref,
"hypothesis": hyp,
"chunk": {"start": excerpt.start, "end": excerpt.end},
"meta": {
"file": filename,
"start": f"{excerpt.start:.1f}",
"end": f"{excerpt.end:.1f}",
},
}
# RTTM files contain one line per speech turn, using the following convention:
# SPEAKER {uri} 1 {start_time} {duration} <NA> <NA> {speaker_id} <NA> <NA>
# * uri: file identifier (as given by pyannote.database protocols)
# * start_time: speech turn start time in seconds
# * duration: speech turn duration in seconds
# * confidence: confidence score (can be anything, not used for now)
# * gender: speaker gender (can be anything, not used for now)
# * speaker_id: speaker identifier
@prodigy.recipe(
"pyannote.diff",
dataset=("Dataset to save annotations to", "positional", None, str),
source=(
"Path to directory containing audio files whose annotation is to be checked",
"positional",
None,
Path,
),
reference=(
"Path to reference file using the RTTM file format",
"positional",
None,
str,
),
hypothesis=(
"Path to hypothesis file using the RTTM file format",
"positional",
None,
str,
),
chunk=(
"Split long audio files into shorter chunks of that many seconds each",
"option",
None,
float,
),
min_duration=("Minimum duration of errors in ms", "option", None, int),
diarization=(
"Optimal one-to-one mapping between reference and hypothesis",
"flag",
None,
bool,
),
false_alarm=("Display false alarm errors", "flag", None, bool),
speaker_confusion=("Display confusion errors", "flag", None, bool),
missed_detection=("Display missed detection errors", "flag", None, bool),
)
def diff(
dataset: str,
source: Path,
reference: str,
hypothesis: str,
chunk: float = 30.0,
min_duration=200,
diarization: bool = False,
false_alarm: bool = False,
speaker_confusion: bool = False,
missed_detection: bool = False,
) -> Dict[str, Any]:
recipe_dir = Path(__file__).resolve().parent
common_dir = recipe_dir.parent / "common"
controllerDiff = recipe_dir / "controller.js"
legend = recipe_dir / "legend.html"
wavesurfer = common_dir / "wavesurfer.js"
regions = common_dir / "regions.js"
html = common_dir / "template.html"
css = common_dir / "template.css"
with open(controllerDiff) as sc_diff, open(wavesurfer) as s_wavesurfer, open(
regions
) as s_regions, open(html) as f_html, open(legend) as f_legend, open(css) as f_css:
script_text = s_wavesurfer.read()
script_text += "\n" + s_regions.read()
script_text += "\n" + sc_diff.read()
templateH = f_html.read()
templateL = f_legend.read()
templateC = f_css.read()
ref = util.load_rttm(reference)
hyp = util.load_rttm(hypothesis)
list_errors = [false_alarm, speaker_confusion, missed_detection]
hashed_stream = (
set_hashes(eg, input_keys=("path", "chunk"))
for eg in diff_stream(
source,
ref,
hyp,
list_errors,
diarization=diarization,
chunk=chunk,
min_duration=min_duration,
)
)
return {
"view_id": "blocks",
"dataset": dataset,
"stream": hashed_stream,
"before_db": before_db,
"config": {
"global_css": templateC,
"javascript": script_text,
"show_audio_minimap": False,
"audio_bar_width": 0,
"audio_bar_height": 1,
"custom_theme": {
"cardMinHeight": 400,
"labels": {
"false alarm": "#9932cc",
"confusion": "#ff6347",
"missed detection": "#00ffff",
},
"palettes": {
"audio": [
"#ffd700",
"#00ffff",
"#ff00ff",
"#00ff00",
"#9932cc",
"#00bfff",
"#ff7f50",
"#66cdaa",
],
},
},
"blocks": [
{"view_id": "html", "html_template": templateL},
{"view_id": "audio"},
{"view_id": "html", "html_template": templateH},
],
"show_audio_timeline": True,
"buttons": ["accept", "ignore", "undo"],
"keymap": {
"accept": ["enter"],
"ignore": ["escape"],
"undo": ["u"],
"playpause": ["space"],
},
"show_flag": True,
},
}
@@ -1,231 +0,0 @@
# The MIT License (MIT)
#
# Copyright (c) 2021-2022 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# AUTHORS
# Jim Petiot
# Hervé Bredin
import base64
import random
from collections.abc import Iterator
from copy import deepcopy
from pathlib import Path
from tempfile import mkstemp
from typing import Any, Dict, Iterable, List, Union
import prodigy
from prodigy import set_hashes
from pyannote.core import Annotation, Segment
from pyannote.audio import Audio, Pipeline
from ..common.utils import AudioForProdigy, before_db, get_audio_spans, get_chunks
def stream(
pipeline: Pipeline,
source: Path,
labels: List[str],
chunk: float = 30.0,
randomize: bool = False,
) -> Iterable[Dict]:
"""Stream for `pyannote.audio` recipe
Applies pretrained pipeline and sends the results for manual correction
Parameters
----------
pipeline : Pipeline
Pretrained pipeline.
source : Path
Directory containing audio files to process.
labels : list of string
List of expected pipeline labels.
chunk : float, optional
Duration of chunks, in seconds. Defaults to 30s.
Yields
------
task : dict
Prodigy task with the following keys:
"path" : path to audio file
"chunk" : chunk start and end times
"audio" : base64 encoding of audio chunk
"text" : chunk identifier "{filename} [{start} {end}]"
"audio_spans" : list of audio spans {"start": ..., "end": ..., "label": ...}
"audio_spans_original" : deep copy of "audio_spans"
"meta" : metadata displayed in Prodigy UI {"file": ..., "start": ..., "end": ...}
"config": {"labels": list of labels}
"""
context = getattr(pipeline, "context", 2.5)
audio_for_prodigy = AudioForProdigy()
audio_for_pipeline = Audio(mono="downmix")
chunks = get_chunks(source, chunk_duration=chunk)
if randomize:
chunks = list(chunks)
random.shuffle(chunks)
for file, excerpt in chunks:
path = file["path"]
filename = file["text"]
text = f"{filename} [{excerpt.start:.1f} - {excerpt.end:.1f}]"
# load contextualized audio excerpt
excerpt_with_context = Segment(
start=excerpt.start - context, end=excerpt.end + context
)
waveform_with_context, sample_rate = audio_for_pipeline.crop(
path, excerpt_with_context, mode="pad"
)
# run pipeline on contextualized audio excerpt
output: Annotation = pipeline(
{"waveform": waveform_with_context, "sample_rate": sample_rate}
)
# crop, shift, and format output for visualization in Prodigy
audio_spans = get_audio_spans(
output, excerpt, excerpt_with_context=excerpt_with_context
)
# load audio excerpt for visualization in Prodigy
audio = audio_for_prodigy.crop(path, excerpt)
yield {
"path": path,
"text": text,
"audio": audio,
"audio_spans": audio_spans,
"audio_spans_original": deepcopy(audio_spans),
"chunk": {"start": excerpt.start, "end": excerpt.end},
"config": {"labels": sorted(set(output.labels()) | set(labels))},
"meta": {
"file": filename,
"start": f"{excerpt.start:.1f}",
"end": f"{excerpt.end:.1f}",
},
}
@prodigy.recipe(
"pyannote.audio",
dataset=("Dataset to save annotations to", "positional", None, str),
source=(
"Path to directory containing audio files to annotate",
"positional",
None,
Path,
),
pipeline=(
"Pretrained pipeline (name on Huggingface Hub, path to YAML file)",
"positional",
None,
str,
),
chunk=(
"Split long audio files into shorter chunks of that many seconds each",
"option",
None,
float,
),
num_classes=(
"Set maximum number of classes for pipelines whose number of classes is not predefined (e.g. pyannote/speaker-diarization)",
"option",
None,
int,
),
precision=("Keyboard temporal precision, in milliseconds.", "option", None, int),
beep=(
"Beep when the player reaches the end of a region.",
"flag",
None,
bool,
),
)
def pipeline(
dataset: str,
source: Path,
pipeline: Union[str, Iterable[dict]],
chunk: float = 10.0,
num_classes: int = 4,
precision: int = 200,
beep: bool = False,
) -> Dict[str, Any]:
pipeline = Pipeline.from_pretrained(pipeline, use_auth_token=True)
classes = pipeline.classes()
if isinstance(classes, Iterator):
labels = [x for _, x in zip(range(num_classes), classes)]
else:
labels = classes
recipe_dir = Path(__file__).resolve().parent
common_dir = recipe_dir.parent / "common"
controller_js = common_dir / "controller.js"
with open(controller_js) as txt:
javascript = txt.read()
# TODO: improve this part
template = common_dir / "instructions.html"
png = common_dir / "commands.png"
_, instructions_html = mkstemp(text=True)
with open(instructions_html, "w") as instructions_f, open(
template, "r"
) as fp_tpl, open(png, "rb") as fp_png:
b64 = base64.b64encode(fp_png.read()).decode("utf-8")
instructions_f.write(fp_tpl.read().replace("{IMAGE}", b64))
hashed_stream = (
set_hashes(eg, input_keys=("path", "chunk"))
for eg in stream(pipeline, source, labels, chunk=chunk, randomize=False)
)
return {
"view_id": "audio_manual",
"dataset": dataset,
"stream": hashed_stream,
"before_db": before_db,
"config": {
"javascript": javascript,
"instructions": instructions_html,
"buttons": ["accept", "ignore", "undo"],
"keymap": {
"accept": ["enter"],
"ignore": ["escape"],
"undo": ["u"],
"playpause": ["space"],
},
"show_audio_minimap": False,
"audio_autoplay": True,
"audio_bar_width": 0,
"audio_bar_height": 1,
"show_flag": True,
"labels": labels,
"precision": precision,
"beep": beep,
},
}
@@ -1,228 +0,0 @@
// The MIT License (MIT)
//
// Copyright (c) 2021 CNRS
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
var numberAnnotations = prodigy.config.number_annotations;
var colorList = [];
var hex2rgba = (hex, alpha = 0.2) => {
const [r, g, b] = hex.match(/\w\w/g).map(x => parseInt(x, 16));
return `rgba(${r},${g},${b},${alpha})`;
};
/**
* Makes sure that the document is loaded before executing wait()
*/
if (document.readyState !== 'loading') {
wait();
} else {
document.addEventListener('DOMContentLoaded', function () {
wait();
});
}
/**
* Add regions for all annotations on their corresponding wavesurfer instance
*/
function loadRegions() {
for (var i = 0; i < numberAnnotations; i++) {
var regions = window.prodigy.content.annotations[i];
for (region in regions) {
var color;
var label = regions[region]['label'];
if (!colorList.includes(label)) {
colorList.push(label);
}
var id = colorList.indexOf(label) % prodigy.config.custom_theme.palettes.audio.length;
var color = prodigy.config.custom_theme.palettes.audio[id];
color = hex2rgba(color);
var re = window['wavesurfer' + i].addRegion({ 'start': regions[region]['start'], 'end': regions[region]['end'], 'color': color, 'resize': false, 'drag': false, "attributes": { "label": label } });
addRegionLabel(re, label);
}
}
}
/**
* Create audio url
* @see loadWave()
* @see prodigyanswer
*/
async function createURL() {
var blob;
var track = document.querySelector('#track');
var src = track.children[0].src
var blob = await (await fetch(src)).blob()
var objectURL = URL.createObjectURL(blob);
return objectURL;
}
/**
* Create the same wavesurfer object as on Prodigy interface for all annotations
* Those objects are muted
*/
async function loadWave() {
var objectURL = await createURL();
var wdict = {
container: window.wavesurfer.container,
audioRate: 1,
autoCenter: true,
autoCenterImmediately: false,
backend: "WebAudio",
barGap: 2,
barHeight: 1,
barMinHeight: null,
barRadius: 2,
barWidth: 0,
cursorColor: "#333",
cursorWidth: 1,
fillParent: true,
forceDecode: false,
height: 128,
hideScrollbar: false,
interact: false,
loopSelection: true,
maxCanvasWidth: 4000,
mediaControls: false,
mediaType: "audio",
normalize: false,
partialRender: false,
pixelRatio: 2,
progressColor: "#583fcf",
waveColor: "violet",
removeMediaElementOnDestroy: true,
responsive: false,
scrollParent: true,
skipLength: 2,
splitChannels: false,
plugins: [
WaveSurfer.regions.create({})
]
};
for (var i = 0; i < numberAnnotations; i++) {
window['wavesurfer' + i] = WaveSurfer.create(wdict);
window['wavesurfer' + i].load(objectURL);
window['wavesurfer' + i].setMute(true);
window['wavesurfer' + i].on('region-click', function (e) {
var label = e.attributes.label;
clickOnLabel(label);
var re = window.wavesurfer.addRegion({ 'start': e.start, 'end': e.end, 'color': e.color });
window.wavesurfer.fireEvent('region-update-end', re);
});
}
var l = document.querySelector('wave').appendChild(document.createElement("span"));
l.textContent = "Output";
l.className = "title-wave";
const nodeList = document.querySelectorAll('wave ~ wave');
for (var i = 0; i < nodeList.length; i++) {
nodeList[i].style.marginTop = "30px";
nodeList[i].style.backgroundColor = "#0000000a";
l = nodeList[i].appendChild(document.createElement("span"));
l.textContent = "Input " + (i + 1);
l.className = "title-wave";
l.onclick = function () { test(i) };
l.setAttribute("onclick", "addAllRegions(" + i + ");");
}
}
/**
* Handle click on annotation label
* Add all regions of the clicked annotation on the main wavesurfer instance (the prodigy's one)
* @param(i) indice of annotation (wavesurfer instance)
*/
function addAllRegions(i) {
var rs = window['wavesurfer' + i].regions.list;
num = Object.values(rs);
for (var i in num) {
var region = num[i];
var label = region.attributes.label;
clickOnLabel(label);
var re = window.wavesurfer.addRegion({ 'start': region.start, 'end': region.end, 'color': region.color });
window.wavesurfer.fireEvent('region-update-end', re);
}
}
/**
* Handle wavesurfer
* Tracking audio playback for all annotations
*/
async function wait() {
if (document.querySelector('#track') !== null) {
await loadWave();
loadRegions();
window.wavesurfer.on('audioprocess', function (e) {
var time = e / window.wavesurfer.getDuration();
for (var i = 0; i < numberAnnotations; i++) {
window['wavesurfer' + i].seekAndCenter(time);
}
});
window.wavesurfer.on('seek', function (e) {
for (var i = 0; i < numberAnnotations; i++) {
window['wavesurfer' + i].seekTo(e);
}
});
window.wavesurfer.on('zoom', function (e) {
for (var i = 0; i < numberAnnotations; i++) {
window['wavesurfer' + i].zoom(e);
}
});
window.wavesurfer.on('finish', function (e) {
for (var i = 0; i < numberAnnotations; i++) {
window['wavesurfer' + i].seekTo(1);
}
});
window.wavesurfer.on('pause', function (e) {
for (var i = 0; i < numberAnnotations; i++) {
window['wavesurfer' + i].pause();
}
});
} else {
setTimeout(wait, 250);
}
}
/**
* Load new audio and clear regions for all annotations wavesurfer when there is a new task
*/
document.addEventListener('prodigyanswer', async () => {
var objectURL = await createURL();
for (var i = 0; i < numberAnnotations; i++) {
window['wavesurfer' + i].load(objectURL);
window['wavesurfer' + i].clearRegions();
}
loadRegions();
});
/**
* Add CSS label to reference and hypothesis region
* @see loadRegions()
* @param(r) region object
* @param(l) label
*/
function addRegionLabel(r, label) {
var s = r.element
var l = s.appendChild(document.createElement("span"))
l.textContent = label,
l.className = "pyannote-region",
l.style.color = "rgb(0, 0, 0)",
l.style.background = r.color,
r.label = label;
}
-220
View File
@@ -1,220 +0,0 @@
import base64
from pathlib import Path
from tempfile import mkstemp
from typing import Any, Dict, Iterable, List
import prodigy
from prodigy import set_hashes
from prodigy.util import split_string
from pyannote.core import Segment
from pyannote.database import util
from pyannote.audio.core.io import Audio
from pyannote.audio.pipelines import SpeakerDiarization
from ..common.utils import (
AudioForProdigy,
before_db,
get_audio_spans,
get_chunks,
source_to_files,
)
def review_stream(
source: Path,
annotations: [dict],
labels: [dict],
diarization: bool = False,
chunk: float = 30.0,
) -> Iterable[Dict]:
files = source_to_files(source)
audio_for_prodigy = AudioForProdigy()
audio_for_pipeline = Audio(mono="downmix")
chunks = get_chunks(source, chunk_duration=chunk)
chunks = list(chunks)
files_annotations = {}
for file in files:
filename = file["text"]
if diarization:
list_annotations = [annotations[0][filename]] + [
SpeakerDiarization.optimal_mapping(
annotations[0][filename], ann[filename]
)
for ann in annotations[1:]
]
labels = [label for ann in list_annotations for label in ann.labels()]
labels = list(dict.fromkeys(labels))
files_annotations[filename] = (list_annotations, labels)
else:
list_annotations = [ann[filename] for ann in annotations]
files_annotations[filename] = (list_annotations, labels)
for file, excerpt in chunks:
path = file["path"]
filename = file["text"]
text = f"{filename} [{excerpt.start:.1f} - {excerpt.end:.1f}]"
waveform, sample_rate = audio_for_pipeline.crop(path, excerpt, mode="pad")
audio = audio_for_prodigy.crop(path, excerpt)
duration = audio_for_pipeline.get_duration(path)
list_spans = []
for annotation in files_annotations[filename][0]:
spans = get_audio_spans(annotation, excerpt, Segment(0, duration))
list_spans.append(spans)
labels = files_annotations[filename][1]
yield {
"path": path,
"text": text,
"audio": audio,
"audio_spans": [],
"annotations": list_spans,
"config": {"labels": labels},
"chunk": {"start": excerpt.start, "end": excerpt.end},
"meta": {
"file": filename,
"start": f"{excerpt.start:.1f}",
"end": f"{excerpt.end:.1f}",
},
}
@prodigy.recipe(
"pyannote.review",
dataset=("Dataset to save annotations to", "positional", None, str),
source=(
"Path to directory containing audio files whose annotation is to be checked",
"positional",
None,
Path,
),
annotations=(
"Comma-separated paths to annotation files",
"positional",
None,
split_string,
),
chunk=(
"Split long audio files into shorter chunks of that many seconds each",
"option",
None,
float,
),
diarization=(
"Optimal one-to-one mapping between the first annotation and the others",
"flag",
None,
bool,
),
precision=("Cursor speed", "option", None, int),
beep=("Beep when the player reaches the end of a region.", "flag", None, bool),
)
def review(
dataset: str,
source: Path,
annotations: [List[str]],
chunk: float = 30.0,
diarization: bool = False,
precision: int = 200,
beep: bool = False,
) -> Dict[str, Any]:
recipe_dir = Path(__file__).resolve().parent
common_dir = recipe_dir.parent / "common"
controllerReview = recipe_dir / "controller.js"
controller = common_dir / "controller.js"
wavesurfer = common_dir / "wavesurfer.js"
regions = common_dir / "regions.js"
html = common_dir / "template.html"
css = common_dir / "template.css"
template = common_dir / "instructions.html"
png = common_dir / "commands.png"
_, instructions_html = mkstemp(text=True)
with open(controllerReview) as sc_review, open(wavesurfer) as s_wavesurfer, open(
regions
) as s_regions, open(html) as f_html, open(css) as f_css, open(
controller
) as sc, open(
png, "rb"
) as fp_png, open(
instructions_html, "w"
) as instructions_f, open(
template, "r"
) as fp_tpl:
script_text = s_wavesurfer.read()
script_text += "\n" + s_regions.read()
script_text += "\n" + sc_review.read()
script_text += "\n" + sc.read()
templateH = f_html.read()
templateC = f_css.read()
b64 = base64.b64encode(fp_png.read()).decode("utf-8")
instructions_f.write(fp_tpl.read().replace("{IMAGE}", b64))
list_annotations = [util.load_rttm(annotation) for annotation in annotations]
labels = [
label
for ann in list_annotations
for anno in list(ann.values())
for label in anno.labels()
]
hashed_stream = (
set_hashes(eg, input_keys=("path", "chunk"))
for eg in review_stream(
source, list_annotations, labels, diarization=diarization, chunk=chunk
)
)
return {
"view_id": "blocks",
"dataset": dataset,
"stream": hashed_stream,
"before_db": before_db,
"config": {
"global_css": templateC,
"javascript": script_text,
"instructions": instructions_html,
"precision": precision,
"beep": beep,
"show_audio_minimap": False,
"audio_bar_width": 0,
"audio_bar_height": 1,
"number_annotations": len(annotations),
"labels": labels,
"custom_theme": {
"palettes": {
"audio": [
"#ffd700",
"#00ffff",
"#ff00ff",
"#00ff00",
"#9932cc",
"#00bfff",
"#ff7f50",
"#66cdaa",
],
}
},
"blocks": [
{
"view_id": "audio_manual",
},
{"view_id": "html", "html_template": templateH},
],
"show_audio_timeline": True,
"buttons": ["accept", "ignore", "undo"],
"keymap": {
"accept": ["enter"],
"ignore": ["escape"],
"undo": ["u"],
"playpause": ["space"],
},
"show_flag": True,
},
}
-6
View File
@@ -48,8 +48,6 @@ dev =
pre_commit>=2.7
recommonmark>=0.6
black>=22.3.0
interactive =
prodigy>=1.11.0
cli =
hydra-core >=1.1,<1.2
typer >= 0.4.0,<0.5.0
@@ -60,10 +58,6 @@ console_scripts =
pyannote-audio-train=pyannote.audio.cli.train:train
pyannote-audio-eval=pyannote.audio.cli.evaluate:evaluate
prodigy_recipes =
pyannote.audio = pyannote.audio.interactive.pipeline.recipe:pipeline
pyannote.review = pyannote.audio.interactive.review.recipe:review
pyannote.diff = pyannote.audio.interactive.diff.recipe:diff
[test]
# py.test options when running `python setup.py test`
-151
View File
@@ -1,151 +0,0 @@
# Annotating your own data with 💥 Prodigy
Manually segmenting and labeling audio data is time consuming. For speaker diarization, depending on the required level of precision, it may take more than 10 times the duration of a recording to annotate it.
## Table of content
* [Recipes](#recipes)
* [Keyboard shortcuts](#keyboard-shortcuts)
## Recipes
`pyannote.audio` comes with a bunch of [💥 Prodigy](https://prodi.gy) recipes designed to speed things up a bit.
| Recipe | Usage |
|-----------------------|-------------------------------------------------------|
🦻 `pyannote.audio` | Annotate with a [pretrained pipeline](https://huggingface.co/models?other=pyannote-audio-pipeline) in the loop
🧐 `pyannote.review` | Merge multiple annotations
🤲 `pyannote.diff` | Show differences between two annotations
🗄 `pyannote.database` | Dump annotations as [`pyannote.database`](https://github.com/pyannote/pyannote-database/) protocols
### 🦻 `pyannote.audio` | Annotate with a pretrained pipeline in the loop
```bash
prodigy pyannote.audio dataset /path/to/audio/directory pyannote/speaker-segmentation
```
![pyannote.audio screenshot](./assets/prodigy-pyannote.audio.png)
`pyannote.audio` recipe will stream in `.wav` files in chunks and apply [a pretrained pipeline](https://huggingface.co/models?other=pyannote-audio-pipeline). You can then adjust the regions manually if needed.
<details>
<summary>More options</summary>
```
prodigy pyannote.audio [options] dataset source pipeline
dataset Prodigy dataset to save annotations to.
source Path to directory containing audio files to annotate.
pipeline Name of pretrained pipeline on huggingface.co (e.g.
pyannote/speaker-segmentation) or path to local YAML file.
-chunk DURATION Split audio files into shorter chunks of that many seconds.
Defaults to 10s.
-precision STEP Temporal precision of keyboard controls, in milliseconds.
Defaults to 200ms.
-beep Produce a beep when the player reaches the end of a region.
```
</details>
### 🧐 `pyannote.review` | Merge multiple annotations
```bash
prodigy pyannote.review dataset /path/to/audio/directory input1.rttm,input2.rttm
```
![pyannote.review screenshot](./assets/pyannote.review.PNG)
`pyannote.review` recipe take as many annotation files, using the RTTM file format, as you want and let you compare and choose which ones are best within the same stream as `pyannote.audio` recipe.
Click on a segment of the annotation files to add it to the ouput audio, or on "Input X" to add all segments at once.
<details>
<summary>More options</summary>
```
prodigy pyannote.review [options] dataset source annotations
dataset Prodigy dataset to save annotations to.
source Path to directory containing audio files whose annotation is to be checked.
annotations Comma-separated paths to annotation files.
-chunk DURATION Split audio files into shorter chunks of that many seconds.
Defaults to 30s.
-diarization Make a optimal one-to-one mapping between the first annotation and the others.
-precision STEP Temporal precision of keyboard controls, in milliseconds.
Defaults to 200ms.
-beep Produce a beep when the player reaches the end of a region.
```
</details>
### 🤲 `pyannote.diff` | Show differences between two annotations
```bash
prodigy pyannote.diff dataset /path/to/audio/directory /path/to/reference.rttm /path/to/hypothesis.rttm
```
![pyannote.diff screenshot](./assets/pyannote.diff.PNG)
`pyannote.diff` recipe take one reference file and one hypothesis file, using the RTTM file format, and focus where there are the most errors among missed detections, false alarms and confusions.
You can filter on one or more error types and their minimum duration with the corresponding options.
<details>
<summary>More options</summary>
```
prodigy pyannote.diff [options] dataset source reference hypothesis
dataset Prodigy dataset to save annotations to.
source Path to directory containing audio files whose annotation is to be checked.
reference Path to reference file.
hypothesis Path to hypothesis file.
-chunk DURATION Split audio files into shorter chunks of that many seconds.
Defaults to 30s.
-min-duration DURATION Minimum duration of errors in ms.
Defaults to 200ms.
-diarization Make a optimal one-to-one mapping between reference and hypothesis.
-false-alarm Display false alarm errors.
-speaker-confusion Display confusion errors.
-missed-detection Display missed detection errors.
```
</details>
### 🗄 `pyannote.database` | Dump annotations as `pyannote.database` protocols
Work in progress
## Keyboard shortcuts
Though `pyannote.audio` recipes are built on top of the Prodigy [audio interface](https://prodi.gy/docs/api-interfaces#audio), they provide a bunch of handy additional keyboard shortcuts.
| Shortcut | Description |
|-----------------------------------|--------------------------------------------------|
| `left` / `right` (+ `w`) | Shift player cursor (speed up) |
| `up` / `down` | Switch active region |
| `shift + left` / `shift + right` | Shift active region start time |
| `ctrl + left` / `ctrl + right` | Shift active region end time |
| `shift + up` | Create a new region |
| `shift + down` / `backspace` | Remove active region |
| `spacebar` | Play/pause player |
| `escape` | Ignore this sample |
| `enter` | Validate annotation |
## RTTM file format
RTTM files contain one line per speech turn, using the following convention:
```bash
SPEAKER {uri} 1 {start_time} {duration} <NA> <NA> {speaker_id} <NA> <NA>
```
* uri: file identifier (as given by pyannote.database protocols)
* start_time: speech turn start time in seconds
* duration: speech turn duration in seconds
* confidence: confidence score (can be anything, not used for now)
* gender: speaker gender (can be anything, not used for now)
* speaker_id: speaker identifier