{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "'Debug.SpeakerDiarization.Debug' found in /Users/hbredin/Development/pyannote/pyannote-audio/tests/data/database.yml does not define the 'scope' of speaker labels (file, database, or global). Setting it to 'file'.\n", "'Debug.SpeakerDiarization.Bug1237' found in /Users/hbredin/Development/pyannote/pyannote-audio/tests/data/database.yml does not define the 'scope' of speaker labels (file, database, or global). Setting it to 'file'.\n" ] } ], "source": [ "from pyannote.database import registry, FileFinder\n", "registry.load_database('../tests/data/database.yml')\n", "protocol = registry.get_protocol('Debug.SpeakerDiarization.Debug', \n", " preprocessors={\"audio\": FileFinder()})" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel\n", "from pyannote.audio.models.embedding.debug import SimpleEmbeddingModel\n", "from lightning import Trainer" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from pyannote.audio.utils.preview import listen\n", "listen(next(protocol.train()))" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## Voice activity detection" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "GPU available: True (mps), used: False\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Protocol Debug.SpeakerDiarization.Debug does not precompute the output of torchaudio.info(): adding a 'torchaudio.info' preprocessor for you to speed up dataloaders. See pyannote.database documentation on how to do that yourself.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "TPU available: False, using: 0 TPU cores\n", "IPU available: False, using: 0 IPUs\n", "HPU available: False, using: 0 HPUs\n", "Missing logger folder: /Users/hbredin/Development/pyannote/pyannote-audio/notebook/lightning_logs\n", "\n", " | Name | Type | Params | In sizes | Out sizes \n", "----------------------------------------------------------------------------------------------------------------------\n", "0 | mfcc | MFCC | 0 | [32, 1, 32000] | [32, 1, 40, 161] \n", "1 | lstm | LSTM | 18.9 K | [32, 161, 40] | [[32, 161, 64], [[2, 32, 32], [2, 32, 32]]]\n", "2 | classifier | Linear | 65 | [32, 161, 64] | [32, 161, 1] \n", "3 | activation | Sigmoid | 0 | [32, 161, 1] | [32, 161, 1] \n", "4 | validation_metric | MetricCollection | 0 | ? | ? \n", "----------------------------------------------------------------------------------------------------------------------\n", "19.0 K Trainable params\n", "0 Non-trainable params\n", "19.0 K Total params\n", "0.076 Total estimated model params size (MB)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "37e122a182e544648e65bed1f71ae722", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Sanity Checking: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4e8a349a91e447a49ea0d93cccfd266c", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Training: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "90c5c47fd1ce4ee98f9729258fab9692", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validation: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "`Trainer.fit` stopped: `max_epochs=1` reached.\n" ] } ], "source": [ "from pyannote.audio.tasks import VoiceActivityDetection\n", "vad = VoiceActivityDetection(protocol, duration=2., batch_size=32, num_workers=4)\n", "model = SimpleSegmentationModel(task=vad)\n", "trainer = Trainer(max_epochs=1)\n", "_ = trainer.fit(model)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Overlapped speech detection" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "GPU available: True (mps), used: False\n", "TPU available: False, using: 0 TPU cores\n", "IPU available: False, using: 0 IPUs\n", "HPU available: False, using: 0 HPUs\n", "\n", " | Name | Type | Params | In sizes | Out sizes \n", "----------------------------------------------------------------------------------------------------------------------\n", "0 | mfcc | MFCC | 0 | [32, 1, 32000] | [32, 1, 40, 161] \n", "1 | lstm | LSTM | 18.9 K | [32, 161, 40] | [[32, 161, 64], [[2, 32, 32], [2, 32, 32]]]\n", "2 | classifier | Linear | 65 | [32, 161, 64] | [32, 161, 1] \n", "3 | activation | Sigmoid | 0 | [32, 161, 1] | [32, 161, 1] \n", "4 | validation_metric | MetricCollection | 0 | ? | ? \n", "----------------------------------------------------------------------------------------------------------------------\n", "19.0 K Trainable params\n", "0 Non-trainable params\n", "19.0 K Total params\n", "0.076 Total estimated model params size (MB)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "9a8333c0ebdf41a79482d81e94bf0e76", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Sanity Checking: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "96c422e9449e40fa82a7f274cc6e5e16", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Training: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "1f343adebc5a407ab7b33b5b88094edf", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validation: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "`Trainer.fit` stopped: `max_epochs=1` reached.\n" ] } ], "source": [ "from pyannote.audio.tasks import OverlappedSpeechDetection\n", "ovl = OverlappedSpeechDetection(protocol, duration=2., batch_size=32, num_workers=4)\n", "model = SimpleSegmentationModel(task=ovl)\n", "trainer = Trainer(max_epochs=1)\n", "_ = trainer.fit(model)" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## (Local) speaker diarization" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "GPU available: True (mps), used: False\n", "TPU available: False, using: 0 TPU cores\n", "IPU available: False, using: 0 IPUs\n", "HPU available: False, using: 0 HPUs\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "574568dce865407e912423c55dfb002d", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Output()" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "
\n",
       "
\n" ], "text/plain": [ "\n" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "\n", " | Name | Type | Params | In sizes | Out sizes \n", "----------------------------------------------------------------------------------------------------------------------\n", "0 | mfcc | MFCC | 0 | [32, 1, 32000] | [32, 1, 40, 161] \n", "1 | lstm | LSTM | 18.9 K | [32, 161, 40] | [[32, 161, 64], [[2, 32, 32], [2, 32, 32]]]\n", "2 | classifier | Linear | 195 | [32, 161, 64] | [32, 161, 3] \n", "3 | activation | Sigmoid | 0 | [32, 161, 3] | [32, 161, 3] \n", "4 | validation_metric | MetricCollection | 0 | ? | ? \n", "----------------------------------------------------------------------------------------------------------------------\n", "19.1 K Trainable params\n", "0 Non-trainable params\n", "19.1 K Total params\n", "0.077 Total estimated model params size (MB)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " - 27.32% of all chunks contain no speech at all.\n", " - 66.79% contain 1 speaker or less\n", " - 91.25% contain 2 speakers or less\n", " - 98.04% contain 3 speakers or less\n", " - 100.00% contain 4 speakers or less\n", "Setting `max_speakers_per_chunk` to 3. You can override this value (or avoid this estimation step) by passing `max_speakers_per_chunk=3` to the task constructor.\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "423b6758f1c84f3b8ece70744862fb66", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Sanity Checking: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "e85cb23d204c4876a82a8202edc273c5", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Training: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b8f5929201654acf83fbca919a5ac63b", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validation: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "`Trainer.fit` stopped: `max_epochs=1` reached.\n" ] } ], "source": [ "from pyannote.audio.tasks import SpeakerDiarization\n", "seg = SpeakerDiarization(protocol, duration=2., batch_size=32, num_workers=4)\n", "model = SimpleSegmentationModel(task=seg)\n", "trainer = Trainer(max_epochs=1)\n", "_ = trainer.fit(model)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Speaker embedding" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "GPU available: True (mps), used: False\n", "TPU available: False, using: 0 TPU cores\n", "IPU available: False, using: 0 IPUs\n", "HPU available: False, using: 0 HPUs\n", "Loading Debug.SpeakerDiarization.Debug training labels: 10file [00:00, 363.55file/s]\n", "\n", " | Name | Type | Params | In sizes | Out sizes \n", "----------------------------------------------------------------------------------------------------------------------\n", "0 | mfcc | MFCC | 0 | [32, 1, 32000] | [32, 1, 40, 161] \n", "1 | lstm | LSTM | 18.9 K | [32, 161, 40] | [[32, 161, 64], [[2, 32, 32], [2, 32, 32]]]\n", "2 | loss_func | ArcFaceLoss | 832 | ? | ? \n", "3 | validation_metric | MetricCollection | 0 | ? | ? \n", "----------------------------------------------------------------------------------------------------------------------\n", "19.8 K Trainable params\n", "0 Non-trainable params\n", "19.8 K Total params\n", "0.079 Total estimated model params size (MB)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b34c5cbe6c9c409fb5572b0be9598076", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Sanity Checking: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "71077814aa154ba3b4892e8be13a98fe", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Training: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "`Trainer.fit` stopped: `max_epochs=1` reached.\n" ] } ], "source": [ "from pyannote.audio.tasks import SpeakerEmbedding\n", "emb = SpeakerEmbedding(protocol, duration=2., num_workers=4)\n", "model = SimpleEmbeddingModel(task=emb)\n", "trainer = Trainer(max_epochs=1)\n", "_ = trainer.fit(model)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.13" } }, "nbformat": 4, "nbformat_minor": 4 }