mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
166 KiB
166 KiB
In [17]:
# preparing notebook for visualization purposes
# (only show outputs between t=180s and t=240s)
from pyannote.core import notebook, Segment
notebook.crop = Segment(210, 240)In [3]:
from pyannote.database import get_protocol
ami = get_protocol('AMI.SpeakerDiarization.only_words')In [5]:
from pyannote.audio.tasks import VoiceActivityDetection
vad_task = VoiceActivityDetection(ami, duration=2.0, batch_size=128)In [6]:
from pyannote.audio.models.segmentation import PyanNet
vad_model = PyanNet(task=vad_task, sincnet={'stride': 10})In [7]:
import pytorch_lightning as pl
trainer = pl.Trainer(gpus=1, max_epochs=1)
trainer.fit(vad_model)GPU available: True, used: True TPU available: False, using: 0 TPU cores IPU available: False, using: 0 IPUs LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] | Name | Type | Params | In sizes | Out sizes ----------------------------------------------------------------------------------------------------------------------- 0 | sincnet | SincNet | 42.6 K | [128, 1, 32000] | [128, 60, 115] 1 | lstm | LSTM | 589 K | [128, 115, 60] | [[128, 115, 256], [[4, 128, 128], [4, 128, 128]]] 2 | linear | ModuleList | 49.4 K | ? | ? 3 | classifier | Linear | 129 | [128, 115, 128] | [128, 115, 1] 4 | activation | Sigmoid | 0 | [128, 115, 1] | [128, 115, 1] 5 | validation_metric | AUROC | 0 | ? | ? ----------------------------------------------------------------------------------------------------------------------- 681 K Trainable params 0 Non-trainable params 681 K Total params 2.728 Total estimated model params size (MB)
HBox(children=(HTML(value='Validation sanity check'), FloatProgress(value=1.0, bar_style='info', layout=Layout…
HBox(children=(HTML(value='Training'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max…
HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…
In [8]:
test_file = next(ami.test())
# here we use a test file provided by the protocol, but it could be any audio file
# e.g. test_file = "/path/to/test.wav".In [18]:
from pyannote.audio import Inference
vad = Inference(vad_model)
vad_probability = vad(test_file)
vad_probabilityOut [18]:
In [19]:
expected_output = test_file["annotation"].get_timeline().support()
expected_outputOut [19]:
In [20]:
from pyannote.audio import Model
pretrained = Model.from_pretrained("pyannote/segmentation")In [23]:
spk_probability = Inference(pretrained, step=2.5)(test_file)
spk_probabilityOut [23]:
In [22]:
test_file["annotation"].discretize(notebook.crop, resolution=0.010)Out [22]:
In [24]:
from pyannote.audio.tasks import Segmentation
seg_task = Segmentation(ami, duration=5.0, max_num_speakers=4)In [15]:
def test(model, protocol, subset="test"):
from pyannote.audio.utils.signal import binarize
from pyannote.audio.utils.metric import DiscreteDiarizationErrorRate
from pyannote.audio.pipelines.utils import get_devices
(device,) = get_devices(needs=1)
metric = DiscreteDiarizationErrorRate()
files = list(getattr(protocol, subset)())
inference = Inference(model, device=device)
for file in files:
reference = file["annotation"]
hypothesis = binarize(inference(file))
uem = file["annotated"]
_ = metric(reference, hypothesis, uem=uem)
return abs(metric)In [16]:
der_pretrained = test(model=pretrained, protocol=ami, subset="test")
print(f"Local DER (pretrained) = {der_pretrained * 100:.1f}%")Local DER (pretrained) = 18.3%
In [29]:
from copy import deepcopy
finetuned = deepcopy(pretrained)
finetuned.task = seg_taskIn [30]:
trainer = pl.Trainer(gpus=1, max_epochs=1)
trainer.fit(finetuned)GPU available: True, used: True TPU available: False, using: 0 TPU cores IPU available: False, using: 0 IPUs LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] | Name | Type | Params | In sizes | Out sizes ------------------------------------------------------------------------------------------------------------------- 0 | sincnet | SincNet | 42.6 K | [32, 1, 80000] | [32, 60, 293] 1 | lstm | LSTM | 1.4 M | [32, 293, 60] | [[32, 293, 256], [[8, 32, 128], [8, 32, 128]]] 2 | linear | ModuleList | 49.4 K | ? | ? 3 | classifier | Linear | 516 | [32, 293, 128] | [32, 293, 4] 4 | activation | Sigmoid | 0 | [32, 293, 4] | [32, 293, 4] 5 | validation_metric | AUROC | 0 | ? | ? ------------------------------------------------------------------------------------------------------------------- 1.5 M Trainable params 0 Non-trainable params 1.5 M Total params 5.892 Total estimated model params size (MB)
HBox(children=(HTML(value='Validation sanity check'), FloatProgress(value=1.0, bar_style='info', layout=Layout…
HBox(children=(HTML(value='Training'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max…
HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…
In [19]:
der_finetuned = test(model=finetuned, protocol=ami, subset="test")
print(f"Local DER (finetuned) = {der_finetuned * 100:.1f}%")Local DER (finetuned) = 17.5%
In [37]:
Inference('pyannote/segmentation', step=2.5)(test_file)Out [37]:
In [38]:
test_file["annotation"]Out [38]:
In [39]:
from pyannote.audio.tasks import OverlappedSpeechDetection
osd_task = OverlappedSpeechDetection(ami, duration=2.0)
osd_model = Model.from_pretrained("pyannote/segmentation")
osd_model.task = osd_taskIn [40]:
osd_model.freeze_up_to('lstm')Out [40]:
['sincnet', 'sincnet.wav_norm1d', 'sincnet.conv1d', 'sincnet.conv1d.0', 'sincnet.conv1d.0.filterbank', 'sincnet.conv1d.1', 'sincnet.conv1d.2', 'sincnet.pool1d', 'sincnet.pool1d.0', 'sincnet.pool1d.1', 'sincnet.pool1d.2', 'sincnet.norm1d', 'sincnet.norm1d.0', 'sincnet.norm1d.1', 'sincnet.norm1d.2', 'lstm']
In [41]:
trainer = pl.Trainer(gpus=1, max_epochs=1)
trainer.fit(osd_model)GPU available: True, used: True TPU available: False, using: 0 TPU cores IPU available: False, using: 0 IPUs LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] | Name | Type | Params | In sizes | Out sizes ------------------------------------------------------------------------------------------------------------------- 0 | sincnet | SincNet | 42.6 K | [32, 1, 32000] | [32, 60, 115] 1 | lstm | LSTM | 1.4 M | [32, 115, 60] | [[32, 115, 256], [[8, 32, 128], [8, 32, 128]]] 2 | linear | ModuleList | 49.4 K | ? | ? 3 | classifier | Linear | 129 | [32, 115, 128] | [32, 115, 1] 4 | activation | Sigmoid | 0 | [32, 115, 1] | [32, 115, 1] 5 | validation_metric | AUROC | 0 | ? | ? ------------------------------------------------------------------------------------------------------------------- 49.5 K Trainable params 1.4 M Non-trainable params 1.5 M Total params 5.890 Total estimated model params size (MB)
HBox(children=(HTML(value='Validation sanity check'), FloatProgress(value=1.0, bar_style='info', layout=Layout…
HBox(children=(HTML(value='Training'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max…
HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…
In [46]:
from pyannote.audio.utils.signal import binarize
binarize(Inference(osd_model)(test_file))Out [46]:
In [44]:
test_file["annotation"]Out [44]: