mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
Co-authored-by: Hervé BREDIN <hbredin@users.noreply.github.com>
162 KiB
162 KiB
In [ ]:
!pip install -qq speechbrain==0.5.16
!pip install -qq ipython==7.34.0
!pip install -qq ipywidgets openai-whisper whisperx==3.1.5 meeteval
!pip install -qq pyannote.audio[separation]In [1]:
import os
# update ROOT_DIR according to your setup
ROOT_DIR = "path/to/pyannote-audio"
ROOT_DIR = "/Users/hbredin/Development/pyannote/pyannote-audio"
os.environ["ASSET_DIR"] = ROOT_DIR + "/tutorials/assets/separation"In [2]:
# create ASSET_DIR repertory
!mkdir -p ${ASSET_DIR}/sources
# Download audio files from AMI-SDM dataset used in this tutorial
!wget --continue -q -O ${ASSET_DIR}/mixture.wav https://groups.inf.ed.ac.uk/ami/AMICorpusMirror//amicorpus/ES2004a/audio/ES2004a.Mix-Headset.wav
!wget --continue -q -O ${ASSET_DIR}/sources/source0.wav https://groups.inf.ed.ac.uk/ami/AMICorpusMirror//amicorpus/ES2004a/audio/ES2004a.Headset-0.wav
!wget --continue -q -O ${ASSET_DIR}/sources/source1.wav https://groups.inf.ed.ac.uk/ami/AMICorpusMirror//amicorpus/ES2004a/audio/ES2004a.Headset-1.wav
!wget --continue -q -O ${ASSET_DIR}/sources/source2.wav https://groups.inf.ed.ac.uk/ami/AMICorpusMirror//amicorpus/ES2004a/audio/ES2004a.Headset-2.wav
!wget --continue -q -O ${ASSET_DIR}/sources/source3.wav https://groups.inf.ed.ac.uk/ami/AMICorpusMirror//amicorpus/ES2004a/audio/ES2004a.Headset-3.wav
# Download rttm file
!wget --continue -q -O ${ASSET_DIR}/mixture.rttm https://raw.githubusercontent.com/pyannote/AMI-diarization-setup/main/only_words/rttms/test/ES2004a.rttmIn [2]:
# set visualization scope to this specific one-minute segment
from pyannote.core import notebook, Segment
segment = Segment(750, 810)
notebook.crop = segmentIn [3]:
from pyannote.database.util import load_rttm
from pyannote.audio.core.io import Audio
audio = Audio()
uri = "ES2004a" # original file name
file = os.environ["ASSET_DIR"] + "/mixture.wav"
mixture, sample_rate = audio.crop(file=file, segment=segment)
annotations = load_rttm(os.environ["ASSET_DIR"] + "/mixture.rttm")[uri]
annotations = annotations.crop(segment)In [4]:
annotationsOut [4]:
In [ ]:
from pyannote.audio.utils.preview import listen
listen(audio_file=file, segment=segment)In [ ]:
from huggingface_hub import notebook_login
notebook_login()In [ ]:
from pyannote.audio import Pipeline
pipeline = Pipeline.from_pretrained(
"pyannote/speech-separation-ami-1.0", use_auth_token=True)
# you can safely ignore that various warnings below.In [ ]:
pipeline.instantiate(
{
"segmentation": {
"min_duration_off": 0.0,
"threshold": 0.5,
},
"clustering": {
"method": "centroid",
"threshold": 0.68,
"min_cluster_size": 60,
},
"separation": {
"leakage_removal": True,
"asr_collar": 0.0,
}
}
)In [ ]:
import torch
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
pipeline.to(device=device)In [ ]:
from pyannote.metrics.diarization import DiarizationErrorRate
from pyannote.audio.pipelines.utils.hook import ProgressHook
metric = DiarizationErrorRate()
# we use a ProgressHook, which will print progress for each step in the pipeline
with ProgressHook() as hook:
# apply pipeline on the mixture file and get diarization prediction
diarization, sources_hat = pipeline({"waveform": mixture, "sample_rate": sample_rate}, hook=hook)In [13]:
from pyannote.core import Annotation
shifted_diarization = Annotation(uri=uri)
for seg, track, label in diarization.itertracks(yield_label=True):
shifted_segment = Segment(seg.start + 750., seg.end + 750.)
shifted_diarization[shifted_segment, track] = label
In [14]:
shifted_diarizationOut [14]:
In [15]:
metric(annotations, shifted_diarization, detailed=True)Out [15]:
{'confusion': 3.531999999999698,
'total': 69.32000000000005,
'correct': 62.87199999999996,
'missed detection': 2.9160000000003947,
'false alarm': 1.9320000000000164,
'diarization error rate': 0.12088863242931482}In [ ]:
pipeline.instantiate(
{
"segmentation": {"min_duration_off": 0.0, "threshold": 0.82},
"clustering": {
"method": "centroid",
"min_cluster_size": 15,
"threshold": 0.68,
},
"separation": {
"leakage_removal": True,
"asr_collar": 0.32,
}
}
)In [ ]:
import whisperx
from whisper.normalizers import EnglishTextNormalizer
normalizer = EnglishTextNormalizer()
references = [
"So simplification of symbols you could think of. Mm-hmm. Menu, alright. Uh uh Right, I was thinking on the same lines you, instead of having too many b buttons and make it complicated for the user, may h maybe have an L_C_D_ di display or something like that, like a mobile, yeah and with menus. And if it's s somewhat similar to what you have on mobile phone, people might find it easier to browse",
"button or something like that. Yeah. Um. When they're when you've got the main things on the front of it and a section opens up or something to the other functions where you can do sound or options kind of recording, things like that inside it. 'Cause it doesn't make when you pick it up it doesn't make it really complicated to look at, it's obvious what you're doing, um.",
"Mm. Yeah.",
". And symbols that you don't necessarily understand, symbols you're meant to understand that you don't. Oh yeah. Mm-hmm. Mm. Mm-hmm. Actually that just raises a point, I wonder what our design people think, but you know on a mobile phone, you can press a key and it gives you a menu, it's got a menu display, I wonder if incorporating that into the design of a remote control might be useful, so you've got a little L_C_D_ display. With menus, yeah, yeah.",
]
references_formatted = []
for i in range(len(references)):
if references[i] != "":
references_formatted.append(normalizer(references[i]))
references = references_formattedIn [ ]:
from pyannote.audio.pipelines.utils.hook import ProgressHook
with ProgressHook() as hook:
diarization, sources = pipeline({"waveform": mixture, "sample_rate": sample_rate}, hook=hook)In [ ]:
from IPython.display import Audio, display
source_1 = sources[:, 1]
display(Audio(source_1, rate=sample_rate))In [ ]:
import numpy as np
device = "cuda" if torch.cuda.is_available() else "cpu"
compute_type = "float32" # "float16"
modelx = whisperx.load_model("small.en", device=device, compute_type=compute_type)
def apply_whisperx(audio, sample_rate=16000):
audio = np.float32(audio / np.max(np.abs(audio)))
result = modelx.transcribe(audio, batch_size=16, language="en")
output = result["segments"] # after alignment
text = ""
for utterance in output:
text = text + " " + utterance["text"]
return normalizer(text)
predictions = []
for i in range(sources.data.shape[1]):
print(f'Processing source #{i+1}...')
text = apply_whisperx(sources.data[:, i])
predictions.append(text)
# only consider 10 longest predictions to save computation time
predictions = sorted(predictions, key=len, reverse=True)[:10]
if len(predictions) < len(references):
predictions = predictions + [""] * (len(references) - len(predictions))
# normalize prediction
for i in range(len(predictions)):
predictions[i] = normalizer(predictions[i])In [29]:
from itertools import permutations
from meeteval.wer.wer.cp import cp_word_error_rate
min_error_rate = 1.0
all_permutations = list(permutations(predictions, len(references)))
# compute cpwer on each permutation and keep the best one
for permutation in all_permutations:
cpwer = cp_word_error_rate(references, list(permutation))
if cpwer.error_rate < min_error_rate:
min_error_rate = cpwer.error_rate
min_cpwer = cpwer
In [31]:
C = cpwer.length - cpwer.errors
S = cpwer.substitutions
D = cpwer.deletions
I = cpwer.insertions
deletion_rate = D / (C + S + D) * 100
insertion_rate = I / (C + S + D) * 100
substitution_rate = S / (C + S + D) * 100
print("cpWER breakdown")
print(f"Substitution rate: {substitution_rate:.1f}%")
print(f"Deletion rate: {deletion_rate:.1f}%")
print(f"Insertion rate: {insertion_rate:.1f}%")
print(f"Total WER: {cpwer.error_rate * 100:.1f}%")cpWER breakdown Substitution rate: 3.0% Deletion rate: 14.6% Insertion rate: 1.3% Total WER: 18.6%
In [32]:
import numpy as np
chunk = Segment(0, 5)
sources = []
for i in range(3):
file = os.environ["ASSET_DIR"] + f"/sources/source{i}.wav"
source, _ = audio.crop(file=file, segment=chunk, duration=5.)
sources.append(source.squeeze(0).numpy())
sources = np.array(sources)
sources.shape # (num_speakers, num_samples)Out [32]:
(3, 80000)
In [ ]:
from pyannote.audio.core.model import Model
# apply totatonet on the first five seconds
totatonet = Model.from_pretrained("pyannote/separation-ami-1.0", use_auth_token=True)
totatonet.to(device)
cropped_mixture = mixture[:, :sample_rate * 5].unsqueeze(0)
diarization, sources_hat = totatonet(cropped_mixture.to(device))
sources_hat.shape # (batch_size, num_samples, num_sources)In [38]:
# get all possible permutations
sources_hat = sources_hat.squeeze(0).cpu().detach().numpy()
sources_hat = sources_hat.T
all_permutations = list(permutations(sources_hat, 3))In [39]:
# compute metrics on each permutation, and keep the best permutation in terms of SI-SDR
from asteroid.metrics import get_metrics
import numpy as np
best_si_sdr = -1000
metrics_dict = {}
cropped_mixture = cropped_mixture.squeeze().cpu().numpy()
for permutation in all_permutations:
metrics = get_metrics(
metrics_list="all",
mix=cropped_mixture,
clean=sources,
estimate=np.array(permutation)
)
si_sdr = metrics["si_sdr"]
if si_sdr > best_si_sdr:
best_si_sdr = si_sdr
metrics_dict = metrics
metrics_dictOut [39]:
{'input_si_sdr': -65.86242802937825,
'input_sdr': -28.623883950224727,
'input_sir': -4.12515450799696,
'input_sar': -22.447442779085623,
'input_stoi': -0.03292179880781978,
'input_pesq': 1.0474077065785725,
'si_sdr': -45.3333740234375,
'sdr': -23.064030295600308,
'sir': 2.092357678563547,
'sar': -20.517320589163734,
'stoi': -0.03345908977251077,
'pesq': 1.067254622777303}In [40]:
print("SI SDR improvement = ", metrics_dict["si_sdr"] - metrics_dict["input_si_sdr"], "dB")SI SDR improvement = 20.52905400594075 dB