{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"'Debug.SpeakerDiarization.Debug' found in /Users/hbredin/Development/pyannote/pyannote-audio/tests/data/database.yml does not define the 'scope' of speaker labels (file, database, or global). Setting it to 'file'.\n",
"'Debug.SpeakerDiarization.Bug1237' found in /Users/hbredin/Development/pyannote/pyannote-audio/tests/data/database.yml does not define the 'scope' of speaker labels (file, database, or global). Setting it to 'file'.\n"
]
}
],
"source": [
"from pyannote.database import registry, FileFinder\n",
"registry.load_database('../tests/data/database.yml')\n",
"protocol = registry.get_protocol('Debug.SpeakerDiarization.Debug', \n",
" preprocessors={\"audio\": FileFinder()})"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel\n",
"from pyannote.audio.models.embedding.debug import SimpleEmbeddingModel\n",
"from lightning import Trainer"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
],
"text/plain": [
""
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from pyannote.audio.utils.preview import listen\n",
"listen(next(protocol.train()))"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## Voice activity detection"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"GPU available: True (mps), used: False\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Protocol Debug.SpeakerDiarization.Debug does not precompute the output of torchaudio.info(): adding a 'torchaudio.info' preprocessor for you to speed up dataloaders. See pyannote.database documentation on how to do that yourself.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"TPU available: False, using: 0 TPU cores\n",
"IPU available: False, using: 0 IPUs\n",
"HPU available: False, using: 0 HPUs\n",
"Missing logger folder: /Users/hbredin/Development/pyannote/pyannote-audio/notebook/lightning_logs\n",
"\n",
" | Name | Type | Params | In sizes | Out sizes \n",
"----------------------------------------------------------------------------------------------------------------------\n",
"0 | mfcc | MFCC | 0 | [32, 1, 32000] | [32, 1, 40, 161] \n",
"1 | lstm | LSTM | 18.9 K | [32, 161, 40] | [[32, 161, 64], [[2, 32, 32], [2, 32, 32]]]\n",
"2 | classifier | Linear | 65 | [32, 161, 64] | [32, 161, 1] \n",
"3 | activation | Sigmoid | 0 | [32, 161, 1] | [32, 161, 1] \n",
"4 | validation_metric | MetricCollection | 0 | ? | ? \n",
"----------------------------------------------------------------------------------------------------------------------\n",
"19.0 K Trainable params\n",
"0 Non-trainable params\n",
"19.0 K Total params\n",
"0.076 Total estimated model params size (MB)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "37e122a182e544648e65bed1f71ae722",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Sanity Checking: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4e8a349a91e447a49ea0d93cccfd266c",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Training: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "90c5c47fd1ce4ee98f9729258fab9692",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Validation: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"`Trainer.fit` stopped: `max_epochs=1` reached.\n"
]
}
],
"source": [
"from pyannote.audio.tasks import VoiceActivityDetection\n",
"vad = VoiceActivityDetection(protocol, duration=2., batch_size=32, num_workers=4)\n",
"model = SimpleSegmentationModel(task=vad)\n",
"trainer = Trainer(max_epochs=1)\n",
"_ = trainer.fit(model)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Overlapped speech detection"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"GPU available: True (mps), used: False\n",
"TPU available: False, using: 0 TPU cores\n",
"IPU available: False, using: 0 IPUs\n",
"HPU available: False, using: 0 HPUs\n",
"\n",
" | Name | Type | Params | In sizes | Out sizes \n",
"----------------------------------------------------------------------------------------------------------------------\n",
"0 | mfcc | MFCC | 0 | [32, 1, 32000] | [32, 1, 40, 161] \n",
"1 | lstm | LSTM | 18.9 K | [32, 161, 40] | [[32, 161, 64], [[2, 32, 32], [2, 32, 32]]]\n",
"2 | classifier | Linear | 65 | [32, 161, 64] | [32, 161, 1] \n",
"3 | activation | Sigmoid | 0 | [32, 161, 1] | [32, 161, 1] \n",
"4 | validation_metric | MetricCollection | 0 | ? | ? \n",
"----------------------------------------------------------------------------------------------------------------------\n",
"19.0 K Trainable params\n",
"0 Non-trainable params\n",
"19.0 K Total params\n",
"0.076 Total estimated model params size (MB)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "9a8333c0ebdf41a79482d81e94bf0e76",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Sanity Checking: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "96c422e9449e40fa82a7f274cc6e5e16",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Training: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1f343adebc5a407ab7b33b5b88094edf",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Validation: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"`Trainer.fit` stopped: `max_epochs=1` reached.\n"
]
}
],
"source": [
"from pyannote.audio.tasks import OverlappedSpeechDetection\n",
"ovl = OverlappedSpeechDetection(protocol, duration=2., batch_size=32, num_workers=4)\n",
"model = SimpleSegmentationModel(task=ovl)\n",
"trainer = Trainer(max_epochs=1)\n",
"_ = trainer.fit(model)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## (Local) speaker diarization"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"GPU available: True (mps), used: False\n",
"TPU available: False, using: 0 TPU cores\n",
"IPU available: False, using: 0 IPUs\n",
"HPU available: False, using: 0 HPUs\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "574568dce865407e912423c55dfb002d",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Output()"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n"
],
"text/plain": []
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
" | Name | Type | Params | In sizes | Out sizes \n",
"----------------------------------------------------------------------------------------------------------------------\n",
"0 | mfcc | MFCC | 0 | [32, 1, 32000] | [32, 1, 40, 161] \n",
"1 | lstm | LSTM | 18.9 K | [32, 161, 40] | [[32, 161, 64], [[2, 32, 32], [2, 32, 32]]]\n",
"2 | classifier | Linear | 195 | [32, 161, 64] | [32, 161, 3] \n",
"3 | activation | Sigmoid | 0 | [32, 161, 3] | [32, 161, 3] \n",
"4 | validation_metric | MetricCollection | 0 | ? | ? \n",
"----------------------------------------------------------------------------------------------------------------------\n",
"19.1 K Trainable params\n",
"0 Non-trainable params\n",
"19.1 K Total params\n",
"0.077 Total estimated model params size (MB)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - 27.32% of all chunks contain no speech at all.\n",
" - 66.79% contain 1 speaker or less\n",
" - 91.25% contain 2 speakers or less\n",
" - 98.04% contain 3 speakers or less\n",
" - 100.00% contain 4 speakers or less\n",
"Setting `max_speakers_per_chunk` to 3. You can override this value (or avoid this estimation step) by passing `max_speakers_per_chunk=3` to the task constructor.\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "423b6758f1c84f3b8ece70744862fb66",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Sanity Checking: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e85cb23d204c4876a82a8202edc273c5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Training: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b8f5929201654acf83fbca919a5ac63b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Validation: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"`Trainer.fit` stopped: `max_epochs=1` reached.\n"
]
}
],
"source": [
"from pyannote.audio.tasks import SpeakerDiarization\n",
"seg = SpeakerDiarization(protocol, duration=2., batch_size=32, num_workers=4)\n",
"model = SimpleSegmentationModel(task=seg)\n",
"trainer = Trainer(max_epochs=1)\n",
"_ = trainer.fit(model)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Speaker embedding"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"GPU available: True (mps), used: False\n",
"TPU available: False, using: 0 TPU cores\n",
"IPU available: False, using: 0 IPUs\n",
"HPU available: False, using: 0 HPUs\n",
"Loading Debug.SpeakerDiarization.Debug training labels: 10file [00:00, 363.55file/s]\n",
"\n",
" | Name | Type | Params | In sizes | Out sizes \n",
"----------------------------------------------------------------------------------------------------------------------\n",
"0 | mfcc | MFCC | 0 | [32, 1, 32000] | [32, 1, 40, 161] \n",
"1 | lstm | LSTM | 18.9 K | [32, 161, 40] | [[32, 161, 64], [[2, 32, 32], [2, 32, 32]]]\n",
"2 | loss_func | ArcFaceLoss | 832 | ? | ? \n",
"3 | validation_metric | MetricCollection | 0 | ? | ? \n",
"----------------------------------------------------------------------------------------------------------------------\n",
"19.8 K Trainable params\n",
"0 Non-trainable params\n",
"19.8 K Total params\n",
"0.079 Total estimated model params size (MB)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b34c5cbe6c9c409fb5572b0be9598076",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Sanity Checking: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "71077814aa154ba3b4892e8be13a98fe",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Training: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"`Trainer.fit` stopped: `max_epochs=1` reached.\n"
]
}
],
"source": [
"from pyannote.audio.tasks import SpeakerEmbedding\n",
"emb = SpeakerEmbedding(protocol, duration=2., num_workers=4)\n",
"model = SimpleEmbeddingModel(task=emb)\n",
"trainer = Trainer(max_epochs=1)\n",
"_ = trainer.fit(model)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
}
},
"nbformat": 4,
"nbformat_minor": 4
}