mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
239 KiB
239 KiB
In [ ]:
!pip install -qq pyannote.audio==3.1.1
!pip install -qq ipython==7.34.0
!git clone https://github.com/pyannote/AMI-diarization-setup.git
%cd ./AMI-diarization-setup/pyannote/
!bash ./download_ami_mini.sh
%cd /contentIn [1]:
# preparing notebook for visualization purposes
# (only show outputs between t=180s and t=240s)
from pyannote.core import notebook, Segment
notebook.crop = Segment(210, 240)In [ ]:
from pyannote.database import registry, FileFinder
registry.load_database("AMI-diarization-setup/pyannote/database.yml")
ami = registry.get_protocol('AMI.SpeakerDiarization.mini')In [ ]:
from pyannote.audio.tasks import VoiceActivityDetection
vad_task = VoiceActivityDetection(ami, duration=2.0, batch_size=128)In [4]:
from pyannote.audio.models.segmentation import PyanNet
vad_model = PyanNet(task=vad_task, sincnet={'stride': 10})In [5]:
import pytorch_lightning as pl
trainer = pl.Trainer(devices=1, max_epochs=1)
trainer.fit(vad_model)GPU available: False, used: False TPU available: False, using: 0 TPU cores IPU available: False, using: 0 IPUs HPU available: False, using: 0 HPUs | Name | Type | Params | In sizes | Out sizes --------------------------------------------------------------------------------------------------------------------- 0 | sincnet | SincNet | 42.6 K | [1, 1, 32000] | [1, 60, 115] 1 | lstm | LSTM | 589 K | [1, 115, 60] | [[1, 115, 256], [[4, 1, 128], [4, 1, 128]]] 2 | linear | ModuleList | 49.4 K | ? | ? 3 | classifier | Linear | 129 | [1, 115, 128] | [1, 115, 1] 4 | activation | Sigmoid | 0 | [1, 115, 1] | [1, 115, 1] 5 | validation_metric | MetricCollection | 0 | ? | ? --------------------------------------------------------------------------------------------------------------------- 681 K Trainable params 0 Non-trainable params 681 K Total params 2.728 Total estimated model params size (MB)
Sanity Checking: | | 0/? [00:00<?, ?it/s]
Training: | | 0/? [00:00<?, ?it/s]
Validation: | | 0/? [00:00<?, ?it/s]
`Trainer.fit` stopped: `max_epochs=1` reached.
In [6]:
test_file = next(ami.test())
# here we use a test file provided by the protocol, but it could be any audio file
# e.g. test_file = "/path/to/test.wav".In [7]:
from pyannote.audio import Inference
vad = Inference(vad_model)
vad_probability = vad(test_file)
vad_probabilityOut [7]:
In [8]:
expected_output = test_file["annotation"].get_timeline().support()
expected_outputOut [8]:
In [ ]:
from huggingface_hub import notebook_login
notebook_login()In [10]:
from pyannote.audio import Model
pretrained = Model.from_pretrained("pyannote/segmentation-3.0", use_auth_token=True)In [11]:
spk_probability = Inference(pretrained, step=2.5)(test_file)
spk_probabilityOut [11]:
In [12]:
test_file["annotation"].discretize(notebook.crop, resolution=0.010)Out [12]:
In [13]:
from pyannote.audio.tasks import SpeakerDiarization
seg_task = SpeakerDiarization(ami, duration=10.0, max_speakers_per_chunk=3, max_speakers_per_frame=2)In [14]:
def test(model, protocol, subset="test"):
from pyannote.audio.utils.signal import binarize
from pyannote.audio.utils.metric import DiscreteDiarizationErrorRate
from pyannote.audio.pipelines.utils import get_devices
(device,) = get_devices(needs=1)
metric = DiscreteDiarizationErrorRate()
files = list(getattr(protocol, subset)())
inference = Inference(model, device=device)
for file in files:
reference = file["annotation"]
hypothesis = binarize(inference(file))
uem = file["annotated"]
_ = metric(reference, hypothesis, uem=uem)
return abs(metric)In [15]:
der_pretrained = test(model=pretrained, protocol=ami, subset="test")
print(f"Local DER (pretrained) = {der_pretrained * 100:.1f}%")Local DER (pretrained) = 19.7%
In [16]:
from copy import deepcopy
finetuned = deepcopy(pretrained)
finetuned.task = seg_taskIn [17]:
trainer = pl.Trainer(devices=1, max_epochs=1)
trainer.fit(finetuned)GPU available: False, used: False TPU available: False, using: 0 TPU cores IPU available: False, using: 0 IPUs HPU available: False, using: 0 HPUs
| Name | Type | Params | In sizes | Out sizes ---------------------------------------------------------------------------------------------------------------------- 0 | sincnet | SincNet | 42.6 K | [1, 1, 160000] | [1, 60, 589] 1 | lstm | LSTM | 1.4 M | [1, 589, 60] | [[1, 589, 256], [[8, 1, 128], [8, 1, 128]]] 2 | linear | ModuleList | 49.4 K | ? | ? 3 | classifier | Linear | 903 | [1, 589, 128] | [1, 589, 7] 4 | activation | LogSoftmax | 0 | [1, 589, 7] | [1, 589, 7] 5 | powerset | Powerset | 0 | ? | ? 6 | validation_metric | MetricCollection | 0 | ? | ? ---------------------------------------------------------------------------------------------------------------------- 1.5 M Trainable params 0 Non-trainable params 1.5 M Total params 5.893 Total estimated model params size (MB)
Sanity Checking: | | 0/? [00:00<?, ?it/s]
Training: | | 0/? [00:00<?, ?it/s]
Validation: | | 0/? [00:00<?, ?it/s]
`Trainer.fit` stopped: `max_epochs=1` reached.
In [18]:
der_finetuned = test(model=finetuned, protocol=ami, subset="test")
print(f"Local DER (finetuned) = {der_finetuned * 100:.1f}%")Local DER (finetuned) = 18.8%
In [19]:
Inference('pyannote/segmentation-3.0', use_auth_token=True, step=2.5)(test_file)Out [19]:
In [20]:
test_file["annotation"]Out [20]:
In [21]:
from pyannote.audio.tasks import OverlappedSpeechDetection
osd_task = OverlappedSpeechDetection(ami, duration=2.0)
osd_model = Model.from_pretrained("pyannote/segmentation-3.0", use_auth_token=True)
osd_model.task = osd_taskIn [22]:
osd_model.freeze_up_to('lstm')Out [22]:
['sincnet', 'sincnet.wav_norm1d', 'sincnet.conv1d', 'sincnet.conv1d.0', 'sincnet.conv1d.0.filterbank', 'sincnet.conv1d.1', 'sincnet.conv1d.2', 'sincnet.pool1d', 'sincnet.pool1d.0', 'sincnet.pool1d.1', 'sincnet.pool1d.2', 'sincnet.norm1d', 'sincnet.norm1d.0', 'sincnet.norm1d.1', 'sincnet.norm1d.2', 'lstm']
In [23]:
trainer = pl.Trainer(devices=1, max_epochs=1)
trainer.fit(osd_model)GPU available: False, used: False TPU available: False, using: 0 TPU cores IPU available: False, using: 0 IPUs HPU available: False, using: 0 HPUs
| Name | Type | Params | In sizes | Out sizes --------------------------------------------------------------------------------------------------------------------- 0 | sincnet | SincNet | 42.6 K | [1, 1, 32000] | [1, 60, 115] 1 | lstm | LSTM | 1.4 M | [1, 115, 60] | [[1, 115, 256], [[8, 1, 128], [8, 1, 128]]] 2 | linear | ModuleList | 49.4 K | ? | ? 3 | classifier | Linear | 129 | [1, 115, 128] | [1, 115, 1] 4 | activation | Sigmoid | 0 | [1, 115, 1] | [1, 115, 1] 5 | validation_metric | MetricCollection | 0 | ? | ? --------------------------------------------------------------------------------------------------------------------- 49.5 K Trainable params 1.4 M Non-trainable params 1.5 M Total params 5.890 Total estimated model params size (MB)
Sanity Checking: | | 0/? [00:00<?, ?it/s]
Training: | | 0/? [00:00<?, ?it/s]
Validation: | | 0/? [00:00<?, ?it/s]
`Trainer.fit` stopped: `max_epochs=1` reached.
In [24]:
from pyannote.audio.utils.signal import binarize
binarize(Inference(osd_model)(test_file))Out [24]:
In [25]:
test_file["annotation"]Out [25]: