mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
feat: add Debug database and notebook testing
Co-authored-by: Hervé BREDIN <hbredin@users.noreply.github.com>
This commit is contained in:
1 parent
2ee6b47397
commit
5512a1dba7
6 files changed
+75
-141
No files matched your search
@@ -24,6 +24,7 @@ jobs:
|
||||
pip install -e .[dev,testing]
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
export PYANNOTE_DATABASE_CONFIG=$GITHUB_WORKSPACE/tests/data/database.yml
|
||||
pytest
|
||||
# Turn back on when the repository is public again
|
||||
# - name: Generate coverage report
|
||||
|
||||
@@ -80,3 +80,5 @@ gh-pages.pub
|
||||
*.zip
|
||||
.mypy_cache/
|
||||
.vscode/
|
||||
|
||||
**/lightning_logs/**
|
||||
+11
-1
@@ -1,15 +1,26 @@
|
||||
exclude: '^docs/conf.py'
|
||||
|
||||
repos:
|
||||
# Clean Notebooks
|
||||
- repo: https://gitlab.com/iamlikeme/nbhooks
|
||||
rev: 1.0.0 # Set to the most recent version
|
||||
hooks:
|
||||
- id: nb-ensure-clean
|
||||
args: [--meta, ExecuteTime] # Optional blacklist of metadata keys (you can use regex)
|
||||
|
||||
# Format Code
|
||||
- repo: https://github.com/ambv/black
|
||||
rev: stable
|
||||
hooks:
|
||||
- id: black
|
||||
|
||||
# Sort imports
|
||||
- repo: https://github.com/pre-commit/mirrors-isort
|
||||
rev: v4.3.4
|
||||
hooks:
|
||||
- id: isort
|
||||
|
||||
# Formatting, Whitespace, etc
|
||||
- repo: git://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v2.2.3
|
||||
hooks:
|
||||
@@ -27,4 +38,3 @@ repos:
|
||||
args: ['--fix=no']
|
||||
- id: flake8
|
||||
args: ['--ignore=E203,E501,F811,E712,'] # default of Black
|
||||
# F811 is necessary to allow multiple dispatch function redefinition
|
||||
+17
-140
@@ -2,17 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"os.environ['PYANNOTE_DATABASE_CONFIG'] = '/Users/bredin/Development/pyannote/pyannote-audio/tests/data/database.yml'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -30,7 +20,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -40,7 +30,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -50,51 +40,12 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"GPU available: False, used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"\n",
|
||||
" | Name | Type | Params\n",
|
||||
"------------------------------------------\n",
|
||||
"0 | mfcc | MFCC | 0 \n",
|
||||
"1 | lstm | LSTM | 18 K \n",
|
||||
"2 | classifier | Linear | 130 \n",
|
||||
"3 | activation | LogSoftmax | 0 \n",
|
||||
"/Users/bredin/miniconda3/envs/pyannote-audio-v2/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:45: UserWarning: Your `IterableDataset` has `__len__` defined. In combination with multi-processing data loading (e.g. batch size > 1), this can lead to unintended side effects since the samples will be duplicated.\n",
|
||||
" warnings.warn(*args, **kwargs)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "9a872fa9cfcd4e298bb2d9e3410eb4de",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"HBox(children=(HTML(value='Training'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pytorch_lightning as pl\n",
|
||||
"trainer = pl.Trainer(max_epochs=10)\n",
|
||||
"trainer = pl.Trainer(max_epochs=5)\n",
|
||||
"_ = trainer.fit(model, vad)"
|
||||
]
|
||||
},
|
||||
@@ -107,7 +58,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -117,7 +68,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -126,50 +77,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"GPU available: False, used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"\n",
|
||||
" | Name | Type | Params\n",
|
||||
"---------------------------------------\n",
|
||||
"0 | mfcc | MFCC | 0 \n",
|
||||
"1 | lstm | LSTM | 18 K \n",
|
||||
"2 | classifier | Linear | 1 K \n",
|
||||
"3 | activation | Sigmoid | 0 \n",
|
||||
"/Users/bredin/miniconda3/envs/pyannote-audio-v2/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:45: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 4 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
|
||||
" warnings.warn(*args, **kwargs)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "368ad052fb504dd3898bd70701b3997f",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"HBox(children=(HTML(value='Training'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trainer = pl.Trainer(max_epochs=10)\n",
|
||||
"trainer = pl.Trainer(max_epochs=5)\n",
|
||||
"_ = trainer.fit(model, spk)"
|
||||
]
|
||||
},
|
||||
@@ -182,7 +94,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -192,7 +104,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -202,46 +114,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"GPU available: False, used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"\n",
|
||||
" | Name | Type | Params\n",
|
||||
"------------------------------\n",
|
||||
"0 | mfcc | MFCC | 0 \n",
|
||||
"1 | lstm | LSTM | 18 K \n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "fba3a039c408450bac0a9d56381b780b",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"HBox(children=(HTML(value='Training'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trainer = pl.Trainer(max_epochs=10)\n",
|
||||
"trainer = pl.Trainer(max_epochs=5)\n",
|
||||
"_ = trainer.fit(model, emb)"
|
||||
]
|
||||
}
|
||||
@@ -262,7 +139,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.9"
|
||||
"version": "3.7.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
@@ -50,6 +50,7 @@ exclude =
|
||||
testing =
|
||||
pytest>=6.0
|
||||
pytest-cov>=2.10
|
||||
nbconvert
|
||||
|
||||
dev =
|
||||
pre_commit>=2.7
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
from glob import glob
|
||||
|
||||
import nbformat
|
||||
from nbconvert import PythonExporter
|
||||
|
||||
|
||||
def convert_notebook(notebookPath):
|
||||
"""
|
||||
This will convert a notebook into a script
|
||||
that we can run
|
||||
"""
|
||||
with open(notebookPath) as fh:
|
||||
nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
|
||||
|
||||
exporter = PythonExporter()
|
||||
source, meta = exporter.from_notebook_node(nb)
|
||||
lines = source.split("\n")
|
||||
nlines = []
|
||||
|
||||
# Remove Magic %% lines
|
||||
# Remove !ls commands
|
||||
for i in range(len(lines)):
|
||||
if "get_ipython" in lines[i]:
|
||||
continue
|
||||
stripped = lines[i].strip()
|
||||
if len(stripped) > 0 and stripped[0] == "!":
|
||||
continue
|
||||
nlines.append(lines[i])
|
||||
source = "\n".join(nlines)
|
||||
return source, meta
|
||||
|
||||
|
||||
def test_can_run_notebooks():
|
||||
# Search for all notebooks in directory
|
||||
notebooks = glob("**/*.ipynb")
|
||||
for nb in notebooks:
|
||||
src, meta = convert_notebook(nb)
|
||||
try:
|
||||
exec(src)
|
||||
except Exception as e:
|
||||
# Which notebook caused the error
|
||||
print("Notebook: ", nb)
|
||||
raise e
|
||||
Reference in new issue
Block a user