mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
Align database CI and releases; require Python 3.11 (#122)
* Align database CI and releases; fix meta-protocol registry lookup * Refactor import of Literal in protocol.py Removed the import error handling for Literal and directly imported it from typing. * Use uv add for transcription installation in README * Remove and ignore uv.lock
This commit is contained in:
1 parent
37df51de71
commit
d5b07685d6
45 files changed
+1305
-2609
No files matched your search
Binary file not shown.
|
After Width: | Height: | Size: 702 KiB |
@@ -0,0 +1,35 @@
|
||||
name: Build Wheels
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
release-tag:
|
||||
description: Release tag; empty on pull requests
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
# hatch-vcs needs Git history and tags to derive the package version.
|
||||
fetch-depth: 0
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version-file: .python-version
|
||||
- name: Build wheel and source distribution
|
||||
run: uv build
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: python-package-distributions
|
||||
path: dist/
|
||||
if-no-files-found: error
|
||||
retention-days: ${{ inputs.release-tag != '' && 90 || 3 }}
|
||||
@@ -0,0 +1,87 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# lint (PR only)
|
||||
# build ──┬──► test ──────────┬──► publish (release only)
|
||||
# └──► version-guard ─┘
|
||||
# ci-complete collects all required results.
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: ./.github/workflows/lint.yml
|
||||
|
||||
build:
|
||||
uses: ./.github/workflows/build-wheels.yml
|
||||
with:
|
||||
release-tag: ${{ github.event.release.tag_name || '' }}
|
||||
|
||||
test:
|
||||
needs: build
|
||||
uses: ./.github/workflows/test.yml
|
||||
|
||||
version-guard:
|
||||
name: versions
|
||||
if: github.event_name == 'release'
|
||||
needs: build
|
||||
uses: ./.github/workflows/version-guard.yml
|
||||
with:
|
||||
release-tag: ${{ github.event.release.tag_name }}
|
||||
|
||||
# PyPI trusted publishing must run in a top-level workflow.
|
||||
publish:
|
||||
if: github.event_name == 'release'
|
||||
needs: [build, test, version-guard]
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: pypi
|
||||
url: https://pypi.org/project/pyannote-database/
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: python-package-distributions
|
||||
path: dist/
|
||||
- name: Upload distributions to GitHub Release
|
||||
uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2
|
||||
with:
|
||||
tag_name: ${{ github.event.release.tag_name }}
|
||||
files: |
|
||||
dist/*.whl
|
||||
dist/*.tar.gz
|
||||
fail_on_unmatched_files: true
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
|
||||
- name: Upload to PyPI
|
||||
run: >-
|
||||
uv publish --trusted-publishing always
|
||||
--publish-url https://upload.pypi.org/legacy/
|
||||
--check-url https://pypi.org/simple/
|
||||
|
||||
ci-complete:
|
||||
if: always()
|
||||
needs: [lint, build, test, version-guard, publish]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
RESULTS: ${{ join(needs.*.result, ' ') }}
|
||||
steps:
|
||||
- name: Check results
|
||||
run: |
|
||||
for result in $RESULTS; do
|
||||
if [[ "$result" == failure || "$result" == cancelled ]]; then
|
||||
echo "::error::Required job finished with result: $result"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
+47
-25
@@ -1,33 +1,55 @@
|
||||
name: Documentation
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
build-and-deploy:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: uv.lock
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
# hatch-vcs needs Git history and tags to derive the package version.
|
||||
fetch-depth: 0
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version-file: .python-version
|
||||
- name: Install documentation dependencies
|
||||
run: |
|
||||
uv venv
|
||||
uv pip install '.[doc,cli]'
|
||||
- name: Build documentation
|
||||
run: uv run --no-project --python .venv/bin/python sphinx-build -W --keep-going -b html doc/source doc/build/html
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: documentation
|
||||
path: doc/build/html
|
||||
include-hidden-files: true
|
||||
retention-days: 3
|
||||
|
||||
- name: Install the project
|
||||
run: uv sync --extra doc
|
||||
|
||||
- name: Build documentation
|
||||
run: |
|
||||
make --directory=doc html
|
||||
touch ./doc/build/html/.nojekyll
|
||||
- name: Deploy
|
||||
uses: peaceiris/actions-gh-pages@v3
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
publish_dir: ./doc/build/html
|
||||
deploy:
|
||||
if: github.event_name == 'push'
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: documentation
|
||||
path: doc/build/html
|
||||
- uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
publish_dir: doc/build/html
|
||||
@@ -0,0 +1,30 @@
|
||||
name: Lint
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
ruff:
|
||||
name: python
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
# Report lint/format issues without blocking PRs.
|
||||
# Remove continue-on-error once the codebase is clean.
|
||||
- name: Run ruff check
|
||||
uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b # v4.0.0
|
||||
continue-on-error: true
|
||||
with:
|
||||
version: "0.17.1"
|
||||
args: "check ."
|
||||
- name: Run ruff format
|
||||
uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b # v4.0.0
|
||||
continue-on-error: true
|
||||
with:
|
||||
version: "0.17.1"
|
||||
args: "format --check ."
|
||||
@@ -1,125 +0,0 @@
|
||||
name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
|
||||
|
||||
on: push
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build distribution 📦
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: uv.lock
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version-file: ".python-version"
|
||||
- name: Build
|
||||
run: uv build
|
||||
- name: Store the distribution packages
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: python-package-distributions
|
||||
path: dist/
|
||||
|
||||
publish-to-pypi:
|
||||
name: >-
|
||||
Publish Python 🐍 distribution 📦 to PyPI
|
||||
if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
|
||||
needs:
|
||||
- build
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: pypi
|
||||
permissions:
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Download all the dists
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: python-package-distributions
|
||||
path: dist/
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: uv.lock
|
||||
- name: Publish distribution 📦 to PyPI
|
||||
run: uv publish --trusted-publishing always --publish-url https://upload.pypi.org/legacy/
|
||||
|
||||
|
||||
github-release:
|
||||
name: >-
|
||||
Sign the Python 🐍 distribution 📦 with Sigstore
|
||||
and upload them to GitHub Release
|
||||
needs:
|
||||
- publish-to-pypi
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
permissions:
|
||||
contents: write # IMPORTANT: mandatory for making GitHub Releases
|
||||
id-token: write # IMPORTANT: mandatory for sigstore
|
||||
|
||||
steps:
|
||||
- name: Download all the dists
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: python-package-distributions
|
||||
path: dist/
|
||||
- name: Sign the dists with Sigstore
|
||||
uses: sigstore/gh-action-sigstore-python@v3.0.0
|
||||
with:
|
||||
inputs: >-
|
||||
./dist/*.tar.gz
|
||||
./dist/*.whl
|
||||
- name: Create GitHub Release
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: >-
|
||||
gh release create
|
||||
"$GITHUB_REF_NAME"
|
||||
--repo "$GITHUB_REPOSITORY"
|
||||
--notes ""
|
||||
- name: Upload artifact signatures to GitHub Release
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
# Upload to GitHub Release using the `gh` CLI.
|
||||
# `dist/` contains the built packages, and the
|
||||
# sigstore-produced signatures and certificates.
|
||||
run: >-
|
||||
gh release upload
|
||||
"$GITHUB_REF_NAME" dist/**
|
||||
--repo "$GITHUB_REPOSITORY"
|
||||
|
||||
# publish-to-testpypi:
|
||||
# name: Publish Python 🐍 distribution 📦 to TestPyPI
|
||||
# needs:
|
||||
# - build
|
||||
# runs-on: ubuntu-latest
|
||||
#
|
||||
# environment:
|
||||
# name: testpypi
|
||||
#
|
||||
# permissions:
|
||||
# id-token: write # IMPORTANT: mandatory for trusted publishing
|
||||
#
|
||||
# steps:
|
||||
# - name: Download all the dists
|
||||
# uses: actions/download-artifact@v4
|
||||
# with:
|
||||
# name: python-package-distributions
|
||||
# path: dist/
|
||||
# - name: Install uv
|
||||
# uses: astral-sh/setup-uv@v5
|
||||
# with:
|
||||
# enable-cache: true
|
||||
# cache-dependency-glob: uv.lock
|
||||
# - name: Publish distribution 📦 to PyPI
|
||||
# run: uv publish --trusted-publishing always --publish-url https://test.pypi.org/legacy/
|
||||
+47
-24
@@ -1,36 +1,59 @@
|
||||
name: Test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- develop
|
||||
push:
|
||||
branches:
|
||||
- develop
|
||||
- master
|
||||
- release/*
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Test
|
||||
name: Test Python ${{ matrix.python-version }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version:
|
||||
- "3.10"
|
||||
- "3.11"
|
||||
- "3.12"
|
||||
env:
|
||||
UV_PYTHON: ${{ matrix.python-version }}
|
||||
python-version: ["3.11", "3.12", "3.13", "3.14", "3.15", "3.x"]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
|
||||
- id: python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
check-latest: ${{ matrix.python-version == '3.x' }}
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: python-package-distributions
|
||||
path: dist/
|
||||
- name: Install the built wheel with test dependencies
|
||||
run: |
|
||||
uv venv --python "${{ steps.python.outputs.python-path }}"
|
||||
uv pip install "$(echo dist/*.whl)[test]"
|
||||
- name: Run tests against the installed wheel
|
||||
run: uv run --no-project --python .venv/bin/python python -m pytest tests
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
|
||||
- name: Install the project
|
||||
run: uv sync --extra test
|
||||
|
||||
- name: Run tests
|
||||
run: uv run pytest tests
|
||||
optional-features:
|
||||
name: Optional features (Python 3.11)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
|
||||
- id: python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: python-package-distributions
|
||||
path: dist/
|
||||
- name: Install the built wheel with test dependencies
|
||||
run: |
|
||||
uv venv --python "${{ steps.python.outputs.python-path }}"
|
||||
uv pip install "$(echo dist/*.whl)[test,cli,transcription]"
|
||||
- name: Run tests against the installed wheel
|
||||
run: uv run --no-project --python .venv/bin/python python -m pytest tests/test_optional_features.py
|
||||
@@ -0,0 +1,70 @@
|
||||
name: Version Guard
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
release-tag:
|
||||
description: Release tag expected to match the package version
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
not-already-published:
|
||||
name: pyannote-database
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version-file: .python-version
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: python-package-distributions
|
||||
path: dist/
|
||||
- name: Compare distributions with PyPI
|
||||
env:
|
||||
RELEASE_TAG: ${{ inputs.release-tag }}
|
||||
shell: python
|
||||
run: |
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import urlopen
|
||||
from zipfile import ZipFile
|
||||
from email.parser import BytesParser
|
||||
|
||||
|
||||
directory = Path("dist")
|
||||
tag = os.environ["RELEASE_TAG"]
|
||||
wheels = list(directory.glob("*.whl"))
|
||||
sources = list(directory.glob("*.tar.gz"))
|
||||
if len(wheels) != 1 or len(sources) != 1:
|
||||
raise ValueError("Expected exactly one wheel and one source distribution")
|
||||
with ZipFile(wheels[0]) as wheel:
|
||||
metadata_path, = [n for n in wheel.namelist() if n.endswith(".dist-info/METADATA")]
|
||||
metadata = BytesParser().parsebytes(wheel.read(metadata_path))
|
||||
version = metadata["Version"]
|
||||
if metadata["Name"] != "pyannote-database" or version != tag.removeprefix("v"):
|
||||
raise ValueError(f"Release tag {tag!r} does not match pyannote-database {version}")
|
||||
try:
|
||||
with urlopen(f"https://pypi.org/pypi/pyannote-database/{version}/json", timeout=30) as response:
|
||||
published = json.load(response)["urls"]
|
||||
except HTTPError as error:
|
||||
if error.code != 404:
|
||||
raise
|
||||
published = []
|
||||
hashes = {item["filename"]: item["digests"]["sha256"] for item in published}
|
||||
for artifact in wheels + sources:
|
||||
if artifact.name not in hashes:
|
||||
print(f"{artifact.name}: not published yet")
|
||||
elif hashlib.sha256(artifact.read_bytes()).hexdigest() == hashes[artifact.name]:
|
||||
print(f"{artifact.name}: already published with identical contents")
|
||||
else:
|
||||
raise ValueError(f"{artifact.name} already exists with different contents. Use a new version.")
|
||||
@@ -89,3 +89,10 @@ ENV/
|
||||
.ropeproject
|
||||
|
||||
.mypy_cache
|
||||
|
||||
.venv/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
|
||||
# Local dependency resolution
|
||||
uv.lock
|
||||
@@ -0,0 +1,7 @@
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.17.1
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [--fix]
|
||||
- id: ruff-format
|
||||
+1
-1
@@ -1 +1 @@
|
||||
3.10
|
||||
3.11
|
||||
+297
@@ -0,0 +1,297 @@
|
||||
# Changelog
|
||||
|
||||
## Next version
|
||||
|
||||
- BREAKING: require Python 3.11 or newer; Python 3.10 is no longer supported
|
||||
- fix: resolve meta-protocol members using the registry that loaded them
|
||||
- BREAKING: forbid `@` in protocol names (now used to select a version)
|
||||
- BREAKING: reserve `version` key in subsets (now used to store the annotation version)
|
||||
- fix: fix `X` meta-protocols with Python 3.14
|
||||
- fix: `FileFinder` patterns only match files (not directories nor hidden files)
|
||||
- feat: add annotation versions with optional metadata (selected with `Database.Task.Protocol@version`)
|
||||
- feat: add `PYANNOTE_DATABASE_DATA_DIRS` environment variable to set root directories of relative `Databases:` paths
|
||||
|
||||
## Version 6.1.1 (2025-12-07)
|
||||
|
||||
- fix: fix warning about invalid espace sequence
|
||||
|
||||
## Version 6.1.0 (2025-09-19)
|
||||
|
||||
- BREAKING: update `STMLoader` to return `meeteval.io.SegLST` instances
|
||||
|
||||
## Version 6.0.0 (2025-09-09)
|
||||
|
||||
- BREAKING: drop support to `Python` < 3.10
|
||||
- BREAKING: switch to native namespace package
|
||||
- setup: switch to `uv`
|
||||
|
||||
## Version 5.1.3 (2025-01-15)
|
||||
|
||||
- chore: remove deprecated use of `delim_whitespace`
|
||||
- chore: use `importlib.metadata` instead of `pkg_resources`
|
||||
|
||||
## Version 5.1.0 (2024-04-05)
|
||||
|
||||
- BREAKING: switch to python 3.9+ (no support for 3.12 yet)
|
||||
- feat(registry): make registry iterable (yields protocol names)
|
||||
- fix(setup): fix `typer`` dependency
|
||||
|
||||
## Version 5.0.1 (2023-04-21)
|
||||
|
||||
- fix: fix support for numeric values in custom protocols (e.g. `channel: 0`)
|
||||
- improve: do not complain about missing scope for meta protocols
|
||||
|
||||
## Version 5.0.0 (2023-03-27)
|
||||
|
||||
- feat(registry): add pyannote.database.registry to load multiple YAML configuration file
|
||||
- BREAKING: turn pyannote.database.get_{database|protocol|protocols} function into registry methods
|
||||
- BREAKING: remove support for database plugins (via "pyannote.database.database" entrypoint)
|
||||
- feat(protocol): add generic segmentation protocol with "classes" metadata
|
||||
- feat(protocol): add support for label "scope" in speaker diarization protocols
|
||||
- improve(protocol): make ProtocolFile pickable
|
||||
- feat(loader): add support for LAB and STM files
|
||||
- feat(loader): add option to filter RTTM lines by "type"
|
||||
- fix(cli): fix corner case when annotation key is missing
|
||||
|
||||
## Version 4.1.3 (2022-02-11)
|
||||
|
||||
- fix: fix corner case with missing uris in RTTM and UEM files
|
||||
- fix(CI): fix documentation deployment
|
||||
|
||||
## Version 4.1.1 (2021-08-4)
|
||||
|
||||
- fix: fix support for meta verification protocol
|
||||
|
||||
## Version 4.1 (2021-04-07)
|
||||
|
||||
- feat: make {RTTM|UEM}Loader usable as preprocessors
|
||||
- improve: remove the need for underscore prefix in database.yml
|
||||
- fix: fix ProtocolFile.__iter__ (dictionary changed size during iteration)
|
||||
|
||||
## Version 4.0.4 (2020-12-14)
|
||||
|
||||
- fix: fix ProtocolFile.__iter__ (dictionary changed size during iteration)
|
||||
|
||||
## Version 4.0.3 (2020-10-11)
|
||||
|
||||
- fix: make custom protocols pickable
|
||||
|
||||
## Version 4.0.2 (2020-10-02)
|
||||
|
||||
- feat: automagically crop "annotated" to file "duration" when available
|
||||
- fix: fix a bug in SpeakerDiarizationProtocols
|
||||
- improve: avoid loading ProtocolFile (lazy) keys in "get_unique_identifier"
|
||||
|
||||
## Version 4.0.1 (2020-06-26)
|
||||
|
||||
- feat: set "name" attribute in get_protocol
|
||||
- fix: display warning only when precomputed value is modified
|
||||
|
||||
## Version 4.0 (2020-06-15)
|
||||
|
||||
- feat: add support for custom speaker verification protocols
|
||||
- feat: add pyannote.database.loader entrypoint
|
||||
- feat: add pyannote-database CLI
|
||||
- feat: add a few dataloaders (RTTM, UEM, CTM, MAP)
|
||||
- feat: add support for nested ProtocolFile
|
||||
- doc: major documentation update (README and docstrings)
|
||||
- BREAKING: custom protocols must define a "uri" section
|
||||
- BREAKING: remove support for "preprocessors" in Database constructor
|
||||
- BREAKING: remove support for progress bars
|
||||
|
||||
## Version 3.0.1 (2020-03-31)
|
||||
|
||||
- BREAKING (feat): if a "database.yml" file exists in current working directory, it will be used even if PYANNOTE_DATABASE_CONFIG is set to another value.
|
||||
- feat: add support in FileFinder for paths relative to "database.yml"
|
||||
- BREAKING: rename "config_yml" option to "database_yml" in FileFinder
|
||||
- feat: add support in custom protocols for paths relative to "database.yml" (@PaulLerner)
|
||||
- BREAKING (feat): use "annotated" to crop "annotation" in custom protocols (@PaulLerner)
|
||||
- fix: add support for int-like protocol name in custom protocols (@PaulLerner)
|
||||
|
||||
## Version 2.5 (2020-02-04)
|
||||
|
||||
- BREAKING: refactor {current | protocol}_file_iter
|
||||
- BREAKING: only rely on "uri" to decide if a ProtocolFile contains multiple files
|
||||
- BREAKING: deprecate FileFinder.current_file_iter in favor of ProtocolFile.files
|
||||
- BREAKING: deprecate FileFinder.protocol_file_iter in favor of Protocol.files
|
||||
- fix: fix support for lazy preprocessors in {Protocol | ProtocolFile}.files
|
||||
|
||||
## Version 2.4.3 (2020-01-24)
|
||||
|
||||
- fix: fix infinite recursion in "ProtocolFile" lazy evaluation
|
||||
|
||||
## Version 2.4.2 (2020-01-06)
|
||||
|
||||
- fix: add support for int-like database name in custom protocol
|
||||
|
||||
## Version 2.4.1 (2019-12-20)
|
||||
|
||||
- fix: make ProtocolFile thread-safe
|
||||
|
||||
## Version 2.4 (2019-12-17)
|
||||
|
||||
- feat: make preprocessors lazy
|
||||
- fix: pandas would convert a label to NaN ([@PaulLerner](https://github.com/PaulLerner))
|
||||
- feat: setup continuous integration
|
||||
- setup: switch to pyannote.core 3.2
|
||||
|
||||
## Version 2.3.1 (2019-09-04)
|
||||
|
||||
- fix: fix support for MDTM files in `pyannote.database.custom.subset_iter` ([#23](https://github.com/pyannote/pyannote-database/issues/23))
|
||||
|
||||
## Version 2.3 (2019-07-19)
|
||||
|
||||
- feat: add LabelMapper preprocessor ([@MarvinLvn](https://github.com/MarvinLvn))
|
||||
- chore: replace (deprecated) pandas.read_table with pandas.read_csv ([@V-assim](https://github.com/V-assim))
|
||||
- chore: use YAML safe loader ([@V-assim](https://github.com/V-assim))
|
||||
|
||||
## Version 2.2 (2019-06-26)
|
||||
|
||||
- setup: switch to pyannote.core 3.0
|
||||
- feat: add RTTMLoader preprocessor
|
||||
|
||||
## Version 2.1 (2019-04-04)
|
||||
|
||||
- feat: add support for "domain" key in generic protocol
|
||||
|
||||
## Version 2.0 (2019-03-20)
|
||||
|
||||
- BREAKING: change location and format of pyannote.database configuration file
|
||||
- feat: add support for PYANNOTE_DATABASE_CONFIG environment variable
|
||||
|
||||
## Version 1.6 (2019-03-12)
|
||||
|
||||
- feat: add support for [file-based speaker diarization protocols](https://github.com/pyannote/pyannote-database/tree/develop#generic-speaker-diarization-protocols)
|
||||
- setup: switch to pyannote.core 2.1
|
||||
|
||||
## Version 1.5.5 (2018-11-30)
|
||||
|
||||
- fix: fix Collection.files when progress=True
|
||||
|
||||
## Version 1.5.4 (2018-11-14)
|
||||
|
||||
- fix: skip files with no "uri" entry in FileFinder.protocol_file_iter
|
||||
|
||||
## Version 1.5.3 (2018-11-08)
|
||||
|
||||
- fix: fix broken SpeakerVerificationProtocol
|
||||
|
||||
## Version 1.5.1 (2018-10-16)
|
||||
|
||||
- fix: fix support for string preprocessors
|
||||
|
||||
## Version 1.5 (2018-09-25)
|
||||
|
||||
- BREAKING: simplify SpeakerVerificationProtocol with {subset}_trial methods
|
||||
|
||||
## Version 1.4 (2018-07-13)
|
||||
|
||||
- feat: add raw collection protocol
|
||||
|
||||
## Version 1.3.2 (2018-05-16)
|
||||
|
||||
- fix: fix regression introduced in 1.3.1
|
||||
|
||||
## Version 1.3.1 (2018-05-11)
|
||||
|
||||
- fix: fix bug in `FileFinder.protocol_file_iter` with empty iterators
|
||||
|
||||
## Version 1.3 (2018-02-04)
|
||||
|
||||
- feat: add `extra_keys` parameter to `{protocol | current}_file_iter`
|
||||
|
||||
## Version 1.2.1 (2018-02-03)
|
||||
|
||||
- setup: drop support for Python 2
|
||||
- feat: add `protocol_file_iter` and `current_file_iter` to FileFinder
|
||||
- feat: add `get_label_identifier` utility function
|
||||
- fix: fix "get_unique_identifier" when "database" or "channel" is None
|
||||
|
||||
## Version 1.1 (2017-10-13)
|
||||
|
||||
- feat: add speaker identification protocol
|
||||
- feat: add speaker verification protocols
|
||||
- feat: add support for list of uris in FileFinder
|
||||
|
||||
## Version 1.0 (2017-10-02)
|
||||
|
||||
- feat: add support for "meta" protocols
|
||||
- feat: add speaker spotting protocol
|
||||
- setup: switch to pyannote.core 1.1
|
||||
|
||||
## Version 0.12 (2017-06-28)
|
||||
|
||||
- feat: add utility functions at package root
|
||||
- doc: improve documentation
|
||||
- doc: add link to pyannote-db-template repository
|
||||
|
||||
## Version 0.11.2 (2017-03-15)
|
||||
|
||||
- fix: fix a bug with string template preprocessors
|
||||
- doc: improve documentation
|
||||
|
||||
## Version 0.11.1 (2017-01_16)
|
||||
|
||||
- feat: add 'get_protocol' helper function
|
||||
|
||||
## Version 0.11 (2017-01-11)
|
||||
|
||||
- feat: add support for validation on training set to speaker recognition protocols
|
||||
- feat: add 'get_annotated' helper function
|
||||
|
||||
## Version 0.10.2 (2017-01-04)
|
||||
|
||||
- fix: fix bug in FileFinder
|
||||
|
||||
## Version 0.10.1 (2016-12-17)
|
||||
|
||||
- improve: change signature of preprocessor.__call__
|
||||
|
||||
## Version 0.9 (2016-12-14)
|
||||
|
||||
- feat: add "get_unique_identifier" utility function
|
||||
|
||||
## Version 0.8.1 (2016-12-12)
|
||||
|
||||
- fix: fix progress bar support
|
||||
|
||||
## Version 0.8 (2016-12-06)
|
||||
|
||||
- feat: add progress bar support
|
||||
|
||||
## Version 0.7.1 (2016-12-03)
|
||||
|
||||
- fix: add 'yield_name' parameter to speaker recognition generators
|
||||
|
||||
## Version 0.7 (2016-12-02)
|
||||
|
||||
- feat: add speaker recognition protocol
|
||||
|
||||
## Version 0.6.1 (2016-12-02)
|
||||
|
||||
- feat: add FileFinder utility class
|
||||
- fix: fix SpeakerDiarizationProtocol.stats()
|
||||
|
||||
## Version 0.5 (2016-12-01)
|
||||
|
||||
- BREAKING: replace 'medium_template' by (more generic) 'preprocessors'
|
||||
|
||||
## Version 0.4.1 (2016-11-17)
|
||||
|
||||
- fix: rename 'speakers' to 'labels' in statistics dictionary
|
||||
|
||||
## Version 0.4 (2016-10-27)
|
||||
|
||||
- feat: add a method providing global statistics about a subset
|
||||
|
||||
## Version 0.3 (2016-09-22)
|
||||
|
||||
- feat: add support for multiple media
|
||||
|
||||
## Version 0.2 (2016-09-21)
|
||||
|
||||
- feat: add support for 'medium_template' attribute
|
||||
|
||||
## Version 0.1 (2016-09-20)
|
||||
|
||||
- first public version
|
||||
@@ -1,3 +1,5 @@
|
||||

|
||||
|
||||
# pyannote-database
|
||||
|
||||
Reproducible experimental protocols for multimedia (audio, video, text) database.
|
||||
@@ -75,6 +77,7 @@ Since version `5.0`, configuration files must be loaded into the `registry` like
|
||||
|
||||
```python
|
||||
from pyannote.database import registry
|
||||
|
||||
registry.load_database("/path/to/database.yml")
|
||||
```
|
||||
|
||||
@@ -93,9 +96,10 @@ Once loaded in the registry, protocols can be used in Python like this:
|
||||
|
||||
```python
|
||||
from pyannote.database import registry
|
||||
|
||||
registry.load_database("/path/to/database.yml")
|
||||
|
||||
protocol = registry.get_protocol('MyDatabase.Protocol.MyProtocol')
|
||||
protocol = registry.get_protocol("MyDatabase.Protocol.MyProtocol")
|
||||
for resource in protocol.train():
|
||||
print(resource["uri"])
|
||||
filename1
|
||||
@@ -125,6 +129,8 @@ with the content of `database.yml` as follows:
|
||||
|
||||
## Data loaders
|
||||
|
||||
STM transcription loading requires the optional dependencies: `uv add "pyannote.database[transcription]"`.
|
||||
|
||||
The above `MyDatabase.Protocol.MyProtocol` protocol is not very useful as it only allows to iterate over a list of resources with a single `'uri'` key. Metadata can be added to each resource with the following syntax:
|
||||
|
||||
```yaml
|
||||
@@ -135,7 +141,7 @@ The above `MyDatabase.Protocol.MyProtocol` protocol is not very useful as it onl
|
||||
train:
|
||||
uri: lists/train.lst
|
||||
speaker: rttms/train.rttm
|
||||
transcription: ctms/{uri}.ctm
|
||||
transcription: stms/{uri}.stm
|
||||
```
|
||||
|
||||
and the following directory structure:
|
||||
@@ -147,60 +153,61 @@ and the following directory structure:
|
||||
| └── train.lst
|
||||
├── rttms
|
||||
| └── train.rttm
|
||||
└── ctms
|
||||
├── filename1.ctm
|
||||
└── filename2.ctm
|
||||
└── stms
|
||||
├── filename1.stm
|
||||
└── filename2.stm
|
||||
```
|
||||
|
||||
Now, resources have both `'speaker'` and `'transcription'` keys:
|
||||
|
||||
```python
|
||||
from pyannote.database import registry
|
||||
protocol = registry.get_protocol('MyDatabase.Protocol.MyProtocol')
|
||||
|
||||
protocol = registry.get_protocol("MyDatabase.Protocol.MyProtocol")
|
||||
|
||||
for resource in protocol.train():
|
||||
assert "speaker" in resource
|
||||
assert isinstance(resource["speaker"], pyannote.core.Annotation)
|
||||
assert "transcription" in resource
|
||||
assert isinstance(resource["transcription"], spacy.tokens.Doc)
|
||||
assert isinstance(resource["transcription"], meeteval.io.SegLST)
|
||||
```
|
||||
|
||||
What happened exactly? Data loaders were automatically selected based on metadata file suffix:
|
||||
|
||||
- `pyannote.database.loader.RTTMLoader` for `speaker` entry with `.rttm` suffix
|
||||
- `pyannote.database.loader.CTMLoader` for `transcription` entry with `ctm` suffix).
|
||||
- `pyannote.database.loader.STMLoader` for `transcription` entry with `.stm` suffix).
|
||||
|
||||
and used to populate `speaker` and `transcription` keys. In pseudo-code:
|
||||
|
||||
```python
|
||||
# instantiate loader registered with `.rttm` suffix
|
||||
speaker = RTTMLoader('rttms/train.rttm')
|
||||
speaker = RTTMLoader("rttms/train.rttm")
|
||||
|
||||
# entries with {placeholders} serve as path templates
|
||||
transcription_template = 'ctms/{uri}.ctm'
|
||||
transcription_template = "stms/{uri}.stm"
|
||||
|
||||
for resource in protocol.train():
|
||||
# unique resource identifier
|
||||
uri = resource['uri']
|
||||
uri = resource["uri"]
|
||||
|
||||
# only select parts of `rttms/train.rttm` that are relevant to current resource,
|
||||
# convert it into a convenient data structure (here pyannote.core.Annotation),
|
||||
# and assign it to `'speaker'` resource key
|
||||
resource['speaker'] = speaker[uri]
|
||||
# convert it into a convenient data structure (here pyannote.core.Annotation),
|
||||
# and assign it to `'speaker'` resource key
|
||||
resource["speaker"] = speaker(resource)
|
||||
|
||||
# replace placeholders in `transcription` path template
|
||||
ctm = transcription_template.format(uri=uri)
|
||||
stm = transcription_template.format(uri=uri)
|
||||
|
||||
# instantiate loader registered with `.ctm` suffix
|
||||
transcription = CTMLoader(ctm)
|
||||
# instantiate loader registered with `.stm` suffix
|
||||
transcription = STMLoader(stm)
|
||||
|
||||
# only select parts of the `ctms/{uri}.ctm` that are relevant to current resource
|
||||
# only select parts of the `stms/{uri}.stm` that are relevant to current resource
|
||||
# (here, most likely the whole file), convert it into a convenient data structure
|
||||
# (here spacy.tokens.Doc), and assign it to `'transcription'` resource key
|
||||
resource['transcription'] = transcription[uri]
|
||||
# (here meeteval.io.SegLST), and assign it to `'transcription'` resource key
|
||||
resource["transcription"] = transcription(resource)
|
||||
```
|
||||
|
||||
`pyannote.database` provides built-in data loaders for a limited set of file formats: `RTTMLoader` for `.rttm` files, `UEMLoader` for `.uem` files, and `CTMLoader` for `.ctm` files. See [Custom data loaders](#custom-data-loaders) section to learn how to add your own.
|
||||
`pyannote.database` provides built-in data loaders for a limited set of file formats: `RTTMLoader` for `.rttm` files, `UEMLoader` for `.uem` files, and `STMLoader` for `.stm` files. See [Custom data loaders](#custom-data-loaders) section to learn how to add your own.
|
||||
|
||||
## Preprocessors
|
||||
|
||||
@@ -313,10 +320,11 @@ It can the be used in Python like this:
|
||||
|
||||
```python
|
||||
from pyannote.database import registry
|
||||
collection = registry.get_protocol('MyDatabase.Collection.MyCollection')
|
||||
|
||||
collection = registry.get_protocol("MyDatabase.Collection.MyCollection")
|
||||
|
||||
for file in collection.files():
|
||||
print(file["uri"])
|
||||
print(file["uri"])
|
||||
filename1
|
||||
filename2
|
||||
filename3
|
||||
@@ -379,12 +387,13 @@ It can then be used in Python like this:
|
||||
|
||||
```python
|
||||
from pyannote.database import registry
|
||||
protocol = registry.get_protocol('MyDatabase.Segmentation.MyProtocol')
|
||||
|
||||
protocol = registry.get_protocol("MyDatabase.Segmentation.MyProtocol")
|
||||
|
||||
for file in protocol.train():
|
||||
print(file["uri"])
|
||||
assert "annotation" in file
|
||||
assert "annotated" in file
|
||||
print(file["uri"])
|
||||
assert "annotation" in file
|
||||
assert "annotated" in file
|
||||
filename1
|
||||
filename2
|
||||
```
|
||||
@@ -443,12 +452,13 @@ It can then be used in Python like this:
|
||||
|
||||
```python
|
||||
from pyannote.database import registry
|
||||
protocol = registry.get_protocol('MyDatabase.SpeakerDiarization.MyProtocol')
|
||||
|
||||
protocol = registry.get_protocol("MyDatabase.SpeakerDiarization.MyProtocol")
|
||||
|
||||
for file in protocol.train():
|
||||
print(file["uri"])
|
||||
assert "annotation" in file
|
||||
assert "annotated" in file
|
||||
print(file["uri"])
|
||||
assert "annotation" in file
|
||||
assert "annotated" in file
|
||||
filename1
|
||||
filename2
|
||||
```
|
||||
@@ -546,11 +556,16 @@ Select a version by appending `@version` to the protocol name:
|
||||
|
||||
```python
|
||||
from pyannote.database import registry
|
||||
protocol = registry.get_protocol('MyDatabase.SpeakerDiarization.MyProtocol@forced_alignment')
|
||||
|
||||
protocol = registry.get_protocol(
|
||||
"MyDatabase.SpeakerDiarization.MyProtocol@forced_alignment"
|
||||
)
|
||||
|
||||
for file in protocol.train():
|
||||
file["annotation"] # loaded from /path/to/forced_alignment/train.rttm
|
||||
file["version"] # {"name": "forced_alignment", "metadata": {"closing": 0.2, "aligner": "mfa"}}
|
||||
file[
|
||||
"version"
|
||||
] # {"name": "forced_alignment", "metadata": {"closing": 0.2, "aligner": "mfa"}}
|
||||
```
|
||||
|
||||
Without `@version`, the first version listed (here `original`) is used.
|
||||
@@ -589,8 +604,9 @@ This new "meta-protocol" can be used like any other protocol of the (fake) `X` d
|
||||
|
||||
```python
|
||||
from pyannote.database import registry
|
||||
protocol = registry.get_protocol('X.Protocol.MyMetaProtocol')
|
||||
|
||||
|
||||
protocol = registry.get_protocol("X.Protocol.MyMetaProtocol")
|
||||
|
||||
for resource in protocol.train():
|
||||
pass
|
||||
```
|
||||
@@ -628,7 +644,7 @@ One can get a list of tasks using `get_tasks` method:
|
||||
|
||||
### Custom data loaders
|
||||
|
||||
`pyannote.database` provides built-in data loaders for a limited set of file formats: `RTTMLoader` for `.rttm` files, `UEMLoader` for `.uem` files, and `CTMLoader` for `.ctm` files.
|
||||
`pyannote.database` provides built-in data loaders for a limited set of file formats: `RTTMLoader` for `.rttm` files, `UEMLoader` for `.uem` files, and `STMLoader` for `.stm` files.
|
||||
|
||||
In case those are not enough, `pyannote.database` supports the addition of custom data loaders using the `pyannote.database.loader` entry point.
|
||||
|
||||
@@ -641,33 +657,37 @@ Here is an example of a Python package called `your_package` that defines two cu
|
||||
from pyannote.database import ProtocolFile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Ext1Loader:
|
||||
def __init__(self, ext1: Path):
|
||||
print(f'Initializing Ext1Loader with {ext1}')
|
||||
print(f"Initializing Ext1Loader with {ext1}")
|
||||
# your code should obviously do something smarter.
|
||||
# see pyannote.database.loader.RTTMLoader for an example.
|
||||
self.ext1 = ext1
|
||||
|
||||
def __call__(self, current_file: ProtocolFile) -> Text:
|
||||
uri = current_file["uri"]
|
||||
print(f'Processing {uri} with Ext1Loader')
|
||||
print(f"Processing {uri} with Ext1Loader")
|
||||
# your code should obviously do something smarter.
|
||||
# see pyannote.database.loader.RTTMLoader for an example.
|
||||
return f'{uri}.ext1'
|
||||
return f"{uri}.ext1"
|
||||
|
||||
|
||||
class Ext2Loader:
|
||||
def __init__(self, ext2: Path):
|
||||
print(f'Initializing Ext2Loader with {ext2}')
|
||||
print(f"Initializing Ext2Loader with {ext2}")
|
||||
# your code should obviously do something smarter.
|
||||
# see pyannote.database.loader.RTTMLoader for an example.
|
||||
self.ext2 = ext2
|
||||
|
||||
def __call__(self, current_file: ProtocolFile) -> Text:
|
||||
uri = current_file["uri"]
|
||||
print(f'Processing {uri} with Ext2Loader')
|
||||
print(f"Processing {uri} with Ext2Loader")
|
||||
# your code should obviously do something smarter.
|
||||
# see pyannote.database.loader.RTTMLoader for an example.
|
||||
return f'{uri}.ext2'
|
||||
return f"{uri}.ext2"
|
||||
|
||||
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
```
|
||||
|
||||
@@ -796,7 +816,7 @@ It can then be used in Python like this:
|
||||
```python
|
||||
collection = MyCollection()
|
||||
for file in collection.files():
|
||||
print(file["uri"])
|
||||
print(file["uri"])
|
||||
filename1
|
||||
filename2
|
||||
filename3
|
||||
@@ -811,12 +831,16 @@ least one of `train_iter`, `development_iter` and `test_iter` methods:
|
||||
```python
|
||||
class MySpeakerDiarizationProtocol(SpeakerDiarizationProtocol):
|
||||
def train_iter(self) -> Iterator[Dict]:
|
||||
yield {"uri": "filename1",
|
||||
"annotation": Annotation(...),
|
||||
"annotated": Timeline(...)}
|
||||
yield {"uri": "filename2",
|
||||
"annotation": Annotation(...),
|
||||
"annotated": Timeline(...)}
|
||||
yield {
|
||||
"uri": "filename1",
|
||||
"annotation": Annotation(...),
|
||||
"annotated": Timeline(...),
|
||||
}
|
||||
yield {
|
||||
"uri": "filename2",
|
||||
"annotation": Annotation(...),
|
||||
"annotated": Timeline(...),
|
||||
}
|
||||
```
|
||||
|
||||
`{subset}_iter` should return an iterator of dictionnaries with
|
||||
@@ -840,7 +864,7 @@ It can then be used in Python like this:
|
||||
```python
|
||||
protocol = MySpeakerDiarizationProtocol()
|
||||
for file in protocol.train():
|
||||
print(file["uri"])
|
||||
print(file["uri"])
|
||||
filename1
|
||||
filename2
|
||||
```
|
||||
@@ -854,26 +878,24 @@ A speaker verification protocol can be defined programmatically by creating a cl
|
||||
```python
|
||||
class MySpeakerVerificationProtocol(SpeakerVerificationProtocol):
|
||||
def train_iter(self) -> Iterator[Dict]:
|
||||
yield {"uri": "filename1",
|
||||
"annotation": Annotation(...),
|
||||
"annotated": Timeline(...)}
|
||||
yield {"uri": "filename2",
|
||||
"annotation": Annotation(...),
|
||||
"annotated": Timeline(...)}
|
||||
yield {
|
||||
"uri": "filename1",
|
||||
"annotation": Annotation(...),
|
||||
"annotated": Timeline(...),
|
||||
}
|
||||
yield {
|
||||
"uri": "filename2",
|
||||
"annotation": Annotation(...),
|
||||
"annotated": Timeline(...),
|
||||
}
|
||||
|
||||
def train_trial_iter(self) -> Iterator[Dict]:
|
||||
yield {"reference": 1,
|
||||
"file1": ProtocolFile(...),
|
||||
"file2": ProtocolFile(...)}
|
||||
yield {"reference": 0,
|
||||
"file1": {
|
||||
"uri":"filename1",
|
||||
"try_with":Timeline(...)
|
||||
},
|
||||
"file1": {
|
||||
"uri":"filename3",
|
||||
"try_with":Timeline(...)
|
||||
}
|
||||
}
|
||||
yield {"reference": 1, "file1": ProtocolFile(...), "file2": ProtocolFile(...)}
|
||||
yield {
|
||||
"reference": 0,
|
||||
"file1": {"uri": "filename1", "try_with": Timeline(...)},
|
||||
"file1": {"uri": "filename3", "try_with": Timeline(...)},
|
||||
}
|
||||
```
|
||||
|
||||
`{subset}_trial_iter` should return an iterator of dictionnaries with
|
||||
@@ -897,3 +919,16 @@ It can then be used in Python like this:
|
||||
1 filename1 filename2
|
||||
0 filename1 filename3
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
Requires Python 3.11 or newer.
|
||||
|
||||
```bash
|
||||
uv sync --extra test --extra cli --extra transcription
|
||||
uv run pytest tests
|
||||
uvx ruff check .
|
||||
uvx ruff format --check .
|
||||
```
|
||||
|
||||
See the [changelog](https://github.com/pyannote/pyannote-database/blob/main/CHANGELOG.md) for release history.
|
||||
@@ -1,355 +0,0 @@
|
||||
#########
|
||||
Changelog
|
||||
#########
|
||||
|
||||
Develop
|
||||
~~~~~~~
|
||||
|
||||
- BREAKING: forbid `@` in protocol names (now used to select a version)
|
||||
- BREAKING: reserve `version` key in subsets (now used to store the annotation version)
|
||||
- fix: fix `X` meta-protocols with Python 3.14
|
||||
- fix: `FileFinder` patterns only match files (not directories nor hidden files)
|
||||
- feat: add annotation versions with optional metadata (selected with `Database.Task.Protocol@version`)
|
||||
- feat: add `PYANNOTE_DATABASE_DATA_DIRS` environment variable to set root directories of relative `Databases:` paths
|
||||
|
||||
Version 6.1.1 (2025-12-07)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix warning about invalid espace sequence
|
||||
|
||||
Version 6.1.0 (2025-09-19)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- BREAKING: update `STMLoader` to return `meeteval.io.SegLST` instances
|
||||
|
||||
Version 6.0.0 (2025-09-09)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- BREAKING: drop support to `Python` < 3.10
|
||||
- BREAKING: switch to native namespace package
|
||||
- setup: switch to `uv`
|
||||
|
||||
Version 5.1.3 (2025-01-15)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- chore: remove deprecated use of `delim_whitespace`
|
||||
- chore: use `importlib.metadata` instead of `pkg_resources`
|
||||
|
||||
Version 5.1.0 (2024-04-05)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- BREAKING: switch to python 3.9+ (no support for 3.12 yet)
|
||||
- feat(registry): make registry iterable (yields protocol names)
|
||||
- fix(setup): fix `typer`` dependency
|
||||
|
||||
Version 5.0.1 (2023-04-21)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix support for numeric values in custom protocols (e.g. `channel: 0`)
|
||||
- improve: do not complain about missing scope for meta protocols
|
||||
|
||||
Version 5.0.0 (2023-03-27)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat(registry): add pyannote.database.registry to load multiple YAML configuration file
|
||||
- BREAKING: turn pyannote.database.get_{database|protocol|protocols} function into registry methods
|
||||
- BREAKING: remove support for database plugins (via "pyannote.database.database" entrypoint)
|
||||
- feat(protocol): add generic segmentation protocol with "classes" metadata
|
||||
- feat(protocol): add support for label "scope" in speaker diarization protocols
|
||||
- improve(protocol): make ProtocolFile pickable
|
||||
- feat(loader): add support for LAB and STM files
|
||||
- feat(loader): add option to filter RTTM lines by "type"
|
||||
- fix(cli): fix corner case when annotation key is missing
|
||||
|
||||
Version 4.1.3 (2022-02-11)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix corner case with missing uris in RTTM and UEM files
|
||||
- fix(CI): fix documentation deployment
|
||||
|
||||
Version 4.1.1 (2021-08-4)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix support for meta verification protocol
|
||||
|
||||
Version 4.1 (2021-04-07)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: make {RTTM|UEM}Loader usable as preprocessors
|
||||
- improve: remove the need for underscore prefix in database.yml
|
||||
- fix: fix ProtocolFile.__iter__ (dictionary changed size during iteration)
|
||||
|
||||
Version 4.0.4 (2020-12-14)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix ProtocolFile.__iter__ (dictionary changed size during iteration)
|
||||
|
||||
Version 4.0.3 (2020-10-11)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: make custom protocols pickable
|
||||
|
||||
Version 4.0.2 (2020-10-02)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: automagically crop "annotated" to file "duration" when available
|
||||
- fix: fix a bug in SpeakerDiarizationProtocols
|
||||
- improve: avoid loading ProtocolFile (lazy) keys in "get_unique_identifier"
|
||||
|
||||
Version 4.0.1 (2020-06-26)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: set "name" attribute in get_protocol
|
||||
- fix: display warning only when precomputed value is modified
|
||||
|
||||
Version 4.0 (2020-06-15)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add support for custom speaker verification protocols
|
||||
- feat: add pyannote.database.loader entrypoint
|
||||
- feat: add pyannote-database CLI
|
||||
- feat: add a few dataloaders (RTTM, UEM, CTM, MAP)
|
||||
- feat: add support for nested ProtocolFile
|
||||
- doc: major documentation update (README and docstrings)
|
||||
- BREAKING: custom protocols must define a "uri" section
|
||||
- BREAKING: remove support for "preprocessors" in Database constructor
|
||||
- BREAKING: remove support for progress bars
|
||||
|
||||
Version 3.0.1 (2020-03-31)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- BREAKING (feat): if a "database.yml" file exists in current working directory, it will be used even if PYANNOTE_DATABASE_CONFIG is set to another value.
|
||||
- feat: add support in FileFinder for paths relative to "database.yml"
|
||||
- BREAKING: rename "config_yml" option to "database_yml" in FileFinder
|
||||
- feat: add support in custom protocols for paths relative to "database.yml" (@PaulLerner)
|
||||
- BREAKING (feat): use "annotated" to crop "annotation" in custom protocols (@PaulLerner)
|
||||
- fix: add support for int-like protocol name in custom protocols (@PaulLerner)
|
||||
|
||||
Version 2.5 (2020-02-04)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- BREAKING: refactor {current | protocol}_file_iter
|
||||
- BREAKING: only rely on "uri" to decide if a ProtocolFile contains multiple files
|
||||
- BREAKING: deprecate FileFinder.current_file_iter in favor of ProtocolFile.files
|
||||
- BREAKING: deprecate FileFinder.protocol_file_iter in favor of Protocol.files
|
||||
- fix: fix support for lazy preprocessors in {Protocol | ProtocolFile}.files
|
||||
|
||||
Version 2.4.3 (2020-01-24)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix infinite recursion in "ProtocolFile" lazy evaluation
|
||||
|
||||
Version 2.4.2 (2020-01-06)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: add support for int-like database name in custom protocol
|
||||
|
||||
Version 2.4.1 (2019-12-20)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: make ProtocolFile thread-safe
|
||||
|
||||
Version 2.4 (2019-12-17)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: make preprocessors lazy
|
||||
- fix: pandas would convert a label to NaN ([@PaulLerner](https://github.com/PaulLerner))
|
||||
- feat: setup continuous integration
|
||||
- setup: switch to pyannote.core 3.2
|
||||
|
||||
Version 2.3.1 (2019-09-04)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix support for MDTM files in `pyannote.database.custom.subset_iter` ([#23](https://github.com/pyannote/pyannote-database/issues/23))
|
||||
|
||||
Version 2.3 (2019-07-19)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add LabelMapper preprocessor ([@MarvinLvn](https://github.com/MarvinLvn))
|
||||
- chore: replace (deprecated) pandas.read_table with pandas.read_csv ([@V-assim](https://github.com/V-assim))
|
||||
- chore: use YAML safe loader ([@V-assim](https://github.com/V-assim))
|
||||
|
||||
Version 2.2 (2019-06-26)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- setup: switch to pyannote.core 3.0
|
||||
- feat: add RTTMLoader preprocessor
|
||||
|
||||
Version 2.1 (2019-04-04)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add support for "domain" key in generic protocol
|
||||
|
||||
Version 2.0 (2019-03-20)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- BREAKING: change location and format of pyannote.database configuration file
|
||||
- feat: add support for PYANNOTE_DATABASE_CONFIG environment variable
|
||||
|
||||
Version 1.6 (2019-03-12)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add support for [file-based speaker diarization protocols](https://github.com/pyannote/pyannote-database/tree/develop#generic-speaker-diarization-protocols)
|
||||
- setup: switch to pyannote.core 2.1
|
||||
|
||||
Version 1.5.5 (2018-11-30)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix Collection.files when progress=True
|
||||
|
||||
Version 1.5.4 (2018-11-14)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: skip files with no "uri" entry in FileFinder.protocol_file_iter
|
||||
|
||||
Version 1.5.3 (2018-11-08)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix broken SpeakerVerificationProtocol
|
||||
|
||||
Version 1.5.1 (2018-10-16)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix support for string preprocessors
|
||||
|
||||
Version 1.5 (2018-09-25)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- BREAKING: simplify SpeakerVerificationProtocol with {subset}_trial methods
|
||||
|
||||
Version 1.4 (2018-07-13)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add raw collection protocol
|
||||
|
||||
Version 1.3.2 (2018-05-16)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix regression introduced in 1.3.1
|
||||
|
||||
Version 1.3.1 (2018-05-11)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix bug in `FileFinder.protocol_file_iter` with empty iterators
|
||||
|
||||
Version 1.3 (2018-02-04)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add `extra_keys` parameter to `{protocol | current}_file_iter`
|
||||
|
||||
Version 1.2.1 (2018-02-03)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- setup: drop support for Python 2
|
||||
- feat: add `protocol_file_iter` and `current_file_iter` to FileFinder
|
||||
- feat: add `get_label_identifier` utility function
|
||||
- fix: fix "get_unique_identifier" when "database" or "channel" is None
|
||||
|
||||
Version 1.1 (2017-10-13)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add speaker identification protocol
|
||||
- feat: add speaker verification protocols
|
||||
- feat: add support for list of uris in FileFinder
|
||||
|
||||
Version 1.0 (2017-10-02)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add support for "meta" protocols
|
||||
- feat: add speaker spotting protocol
|
||||
- setup: switch to pyannote.core 1.1
|
||||
|
||||
Version 0.12 (2017-06-28)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add utility functions at package root
|
||||
- doc: improve documentation
|
||||
- doc: add link to pyannote-db-template repository
|
||||
|
||||
Version 0.11.2 (2017-03-15)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix a bug with string template preprocessors
|
||||
- doc: improve documentation
|
||||
|
||||
Version 0.11.1 (2017-01_16)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add 'get_protocol' helper function
|
||||
|
||||
Version 0.11 (2017-01-11)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add support for validation on training set to speaker recognition protocols
|
||||
- feat: add 'get_annotated' helper function
|
||||
|
||||
Version 0.10.2 (2017-01-04)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix bug in FileFinder
|
||||
|
||||
Version 0.10.1 (2016-12-17)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- improve: change signature of preprocessor.__call__
|
||||
|
||||
Version 0.9 (2016-12-14)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add "get_unique_identifier" utility function
|
||||
|
||||
Version 0.8.1 (2016-12-12)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: fix progress bar support
|
||||
|
||||
Version 0.8 (2016-12-06)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add progress bar support
|
||||
|
||||
Version 0.7.1 (2016-12-03)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: add 'yield_name' parameter to speaker recognition generators
|
||||
|
||||
Version 0.7 (2016-12-02)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add speaker recognition protocol
|
||||
|
||||
Version 0.6.1 (2016-12-02)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add FileFinder utility class
|
||||
- fix: fix SpeakerDiarizationProtocol.stats()
|
||||
|
||||
Version 0.5 (2016-12-01)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- BREAKING: replace 'medium_template' by (more generic) 'preprocessors'
|
||||
|
||||
Version 0.4.1 (2016-11-17)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- fix: rename 'speakers' to 'labels' in statistics dictionary
|
||||
|
||||
Version 0.4 (2016-10-27)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add a method providing global statistics about a subset
|
||||
|
||||
Version 0.3 (2016-09-22)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add support for multiple media
|
||||
|
||||
Version 0.2 (2016-09-21)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- feat: add support for 'medium_template' attribute
|
||||
|
||||
Version 0.1 (2016-09-20)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- first public version
|
||||
+42
-38
@@ -1,4 +1,3 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# pyannote.database documentation build configuration file, created by
|
||||
# sphinx-quickstart on Tue Jan 24 15:45:55 2017.
|
||||
@@ -18,10 +17,7 @@
|
||||
#
|
||||
|
||||
# allow pyannote.database import
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.path.abspath('../..'))
|
||||
|
||||
from importlib.metadata import version as package_version
|
||||
|
||||
# -- General configuration ------------------------------------------------
|
||||
|
||||
@@ -32,15 +28,16 @@ sys.path.insert(0, os.path.abspath('../..'))
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = ['sphinx.ext.autodoc',
|
||||
'sphinx.ext.napoleon',
|
||||
'sphinx.ext.intersphinx',
|
||||
'sphinx.ext.todo',
|
||||
'sphinx.ext.coverage',
|
||||
'sphinx.ext.mathjax',
|
||||
'sphinx.ext.viewcode',
|
||||
'sphinx.ext.githubpages',
|
||||
]
|
||||
extensions = [
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx.ext.intersphinx",
|
||||
"sphinx.ext.todo",
|
||||
"sphinx.ext.coverage",
|
||||
"sphinx.ext.mathjax",
|
||||
"sphinx.ext.viewcode",
|
||||
"sphinx.ext.githubpages",
|
||||
]
|
||||
|
||||
|
||||
# Napoleon settings
|
||||
@@ -58,31 +55,31 @@ napoleon_use_rtype = True
|
||||
napoleon_use_keyword = True
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
templates_path = ["_templates"]
|
||||
|
||||
# The suffix(es) of source filenames.
|
||||
# You can specify multiple suffix as a list of string:
|
||||
#
|
||||
# source_suffix = ['.rst', '.md']
|
||||
source_suffix = {'.rst': 'restructuredtext'}
|
||||
source_suffix = {".rst": "restructuredtext"}
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = 'index'
|
||||
master_doc = "index"
|
||||
|
||||
# General information about the project.
|
||||
project = u'pyannote.database'
|
||||
copyright = u'2019, CNRS'
|
||||
author = u'Hervé Bredin'
|
||||
project = "pyannote.database"
|
||||
copyright = "2019, CNRS"
|
||||
author = "Hervé Bredin"
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
# built documents.
|
||||
|
||||
import pyannote.database
|
||||
|
||||
# The short X.Y version.
|
||||
version = pyannote.database.__version__.split('+')[0]
|
||||
version = package_version("pyannote-database").split("+")[0]
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
release = pyannote.database.__version__
|
||||
release = package_version("pyannote-database")
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
@@ -97,7 +94,7 @@ language = "en"
|
||||
exclude_patterns = []
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = 'sphinx'
|
||||
pygments_style = "sphinx"
|
||||
|
||||
# If true, `todo` and `todoList` produce output, else they produce nothing.
|
||||
todo_include_todos = True
|
||||
@@ -119,13 +116,13 @@ html_theme = "sphinx_rtd_theme"
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['_static']
|
||||
html_static_path = []
|
||||
|
||||
|
||||
# -- Options for HTMLHelp output ------------------------------------------
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = 'pyannotedatabasedoc'
|
||||
htmlhelp_basename = "pyannotedatabasedoc"
|
||||
|
||||
|
||||
# -- Options for LaTeX output ---------------------------------------------
|
||||
@@ -134,15 +131,12 @@ latex_elements = {
|
||||
# The paper size ('letterpaper' or 'a4paper').
|
||||
#
|
||||
# 'papersize': 'letterpaper',
|
||||
|
||||
# The font size ('10pt', '11pt' or '12pt').
|
||||
#
|
||||
# 'pointsize': '10pt',
|
||||
|
||||
# Additional stuff for the LaTeX preamble.
|
||||
#
|
||||
# 'preamble': '',
|
||||
|
||||
# Latex figure (float) alignment
|
||||
#
|
||||
# 'figure_align': 'htbp',
|
||||
@@ -152,8 +146,13 @@ latex_elements = {
|
||||
# (source start file, target name, title,
|
||||
# author, documentclass [howto, manual, or own class]).
|
||||
latex_documents = [
|
||||
(master_doc, 'pyannotedatabase.tex', u'pyannote.database Documentation',
|
||||
u'Hervé Bredin', 'manual'),
|
||||
(
|
||||
master_doc,
|
||||
"pyannotedatabase.tex",
|
||||
"pyannote.database Documentation",
|
||||
"Hervé Bredin",
|
||||
"manual",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@@ -162,8 +161,7 @@ latex_documents = [
|
||||
# One entry per manual page. List of tuples
|
||||
# (source start file, name, description, authors, manual section).
|
||||
man_pages = [
|
||||
(master_doc, 'pyannotedatabase', u'pyannote.database Documentation',
|
||||
[author], 1)
|
||||
(master_doc, "pyannotedatabase", "pyannote.database Documentation", [author], 1)
|
||||
]
|
||||
|
||||
|
||||
@@ -173,12 +171,18 @@ man_pages = [
|
||||
# (source start file, target name, title, author,
|
||||
# dir menu entry, description, category)
|
||||
texinfo_documents = [
|
||||
(master_doc, 'pyannotedatabase', u'pyannote.database Documentation',
|
||||
author, 'pyannotedatabase', 'One line description of project.',
|
||||
'Miscellaneous'),
|
||||
(
|
||||
master_doc,
|
||||
"pyannotedatabase",
|
||||
"pyannote.database Documentation",
|
||||
author,
|
||||
"pyannotedatabase",
|
||||
"One line description of project.",
|
||||
"Miscellaneous",
|
||||
),
|
||||
]
|
||||
|
||||
intersphinx_mapping = {
|
||||
'python': ('https://docs.python.org/', None),
|
||||
'pyannote.core': ('https://pyannote.github.io/pyannote-core', None),
|
||||
"python": ("https://docs.python.org/3/", None),
|
||||
"pyannote.core": ("https://pyannote.github.io/pyannote-core", None),
|
||||
}
|
||||
@@ -14,7 +14,7 @@ Installation
|
||||
|
||||
::
|
||||
|
||||
$ pip install pyannote.database
|
||||
pip install pyannote.database
|
||||
|
||||
|
||||
API documentation
|
||||
@@ -23,4 +23,4 @@ API documentation
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
changelog
|
||||
Changelog <https://github.com/pyannote/pyannote-database/blob/main/CHANGELOG.md>
|
||||
+8
-2
@@ -5,7 +5,7 @@ readme = "README.md"
|
||||
authors = [
|
||||
{ name = "Hervé BREDIN", email = "herve@pyannote.ai" }
|
||||
]
|
||||
requires-python = ">=3.10"
|
||||
requires-python = ">=3.11"
|
||||
|
||||
dynamic = [
|
||||
"version",
|
||||
@@ -43,7 +43,7 @@ transcription = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling", "hatch-vcs"]
|
||||
requires = ["hatchling==1.32.0", "hatch-vcs"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.version]
|
||||
@@ -56,3 +56,9 @@ packages = ["src/pyannote"]
|
||||
dev = [
|
||||
"ipykernel>=6.29.5",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py311"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E4", "E7", "E9", "F", "I", "UP"]
|
||||
@@ -1,84 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2016-2020 CNRS
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# AUTHORS
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
import versioneer
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
setup(
|
||||
# package
|
||||
namespace_packages=["pyannote"],
|
||||
packages=find_packages(),
|
||||
install_requires=[
|
||||
"pyannote.core >= 4.1",
|
||||
"pyYAML >= 3.12",
|
||||
"pandas >= 0.19",
|
||||
"typer >= 0.12.1",
|
||||
"typing_extensions >= 3.7.4;python_version < '3.8'",
|
||||
],
|
||||
entry_points={
|
||||
"console_scripts": [
|
||||
"pyannote-database=pyannote.database.cli:main",
|
||||
],
|
||||
"pyannote.database.loader": [
|
||||
".rttm = pyannote.database.loader:RTTMLoader",
|
||||
".uem = pyannote.database.loader:UEMLoader",
|
||||
".ctm = pyannote.database.loader:CTMLoader",
|
||||
".map = pyannote.database.loader:MAPLoader",
|
||||
".lab = pyannote.database.loader:LABLoader",
|
||||
".stm = pyannote.database.loader:STMLoader",
|
||||
],
|
||||
},
|
||||
# versioneer
|
||||
version=versioneer.get_version(),
|
||||
cmdclass=versioneer.get_cmdclass(),
|
||||
# PyPI
|
||||
name="pyannote.database",
|
||||
description=("Interface to multimedia databases and experimental protocols"),
|
||||
author="Hervé Bredin",
|
||||
author_email="bredin@limsi.fr",
|
||||
url="http://pyannote.github.io/",
|
||||
classifiers=[
|
||||
"Development Status :: 4 - Beta",
|
||||
"Intended Audience :: Science/Research",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Natural Language :: English",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Topic :: Scientific/Engineering",
|
||||
],
|
||||
extras_require={
|
||||
"testing": ["pytest", "flake8==3.7.9"],
|
||||
"doc": [
|
||||
"matplotlib >= 2.0.0",
|
||||
"Sphinx == 2.2.2",
|
||||
"ipython == 7.16.3",
|
||||
"sphinx_rtd_theme == 0.4.3",
|
||||
],
|
||||
},
|
||||
)
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2016- CNRS
|
||||
@@ -29,29 +26,19 @@
|
||||
|
||||
"""pyannote.database"""
|
||||
|
||||
|
||||
from typing import Optional
|
||||
import importlib.metadata
|
||||
import warnings
|
||||
|
||||
from .registry import registry, LoadingMode
|
||||
|
||||
from .database import Database
|
||||
|
||||
from .protocol.protocol import Protocol
|
||||
from .protocol.protocol import ProtocolFile
|
||||
from .protocol.protocol import Subset
|
||||
from .protocol.protocol import Preprocessors
|
||||
|
||||
from .file_finder import FileFinder
|
||||
from .util import get_annotated
|
||||
from .util import get_unique_identifier
|
||||
from .util import get_label_identifier
|
||||
from .protocol.protocol import Preprocessors, Protocol, ProtocolFile, Subset
|
||||
from .registry import LoadingMode, registry
|
||||
from .util import get_annotated, get_label_identifier, get_unique_identifier
|
||||
|
||||
import importlib.metadata
|
||||
__version__ = importlib.metadata.version("pyannote-database")
|
||||
|
||||
|
||||
def get_protocol(name, preprocessors: Optional[Preprocessors] = None) -> Protocol:
|
||||
def get_protocol(name, preprocessors: Preprocessors | None = None) -> Protocol:
|
||||
"""Get protocol by full name
|
||||
|
||||
name : str
|
||||
@@ -68,8 +55,9 @@ def get_protocol(name, preprocessors: Optional[Preprocessors] = None) -> Protoco
|
||||
Protocol instance
|
||||
"""
|
||||
warnings.warn(
|
||||
"`get_protocol` has been deprecated in favor of `pyannote.database.registry.get_protocol`.",
|
||||
DeprecationWarning)
|
||||
"`get_protocol` has been deprecated in favor of `pyannote.database.registry.get_protocol`.",
|
||||
DeprecationWarning,
|
||||
)
|
||||
return registry.get_protocol(name, preprocessors=preprocessors)
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2020- CNRS
|
||||
@@ -28,20 +25,19 @@
|
||||
# Alexis PLAQUET
|
||||
|
||||
|
||||
import typer
|
||||
from enum import Enum
|
||||
import math
|
||||
from typing import Text
|
||||
from pyannote.database import Database
|
||||
from pyannote.database import registry
|
||||
from pyannote.database.protocol import CollectionProtocol
|
||||
from pyannote.database.protocol import SpeakerDiarizationProtocol
|
||||
from enum import StrEnum
|
||||
|
||||
import typer
|
||||
from pyannote.core import Annotation
|
||||
|
||||
from pyannote.database import Database, registry
|
||||
from pyannote.database.protocol import CollectionProtocol, SpeakerDiarizationProtocol
|
||||
|
||||
app = typer.Typer()
|
||||
|
||||
|
||||
class Task(str, Enum):
|
||||
class Task(StrEnum):
|
||||
Any = "Any"
|
||||
Protocol = "Protocol"
|
||||
Collection = "Collection"
|
||||
@@ -65,7 +61,7 @@ def task(
|
||||
metavar="DATABASE",
|
||||
help="Filter tasks by DATABASE.",
|
||||
case_sensitive=False,
|
||||
)
|
||||
),
|
||||
):
|
||||
"""Print list of tasks"""
|
||||
|
||||
@@ -90,7 +86,11 @@ def protocol(
|
||||
case_sensitive=False,
|
||||
),
|
||||
task: Task = typer.Option(
|
||||
"Any", "--task", "-t", help="Filter protocols by TASK.", case_sensitive=False,
|
||||
"Any",
|
||||
"--task",
|
||||
"-t",
|
||||
help="Filter protocols by TASK.",
|
||||
case_sensitive=False,
|
||||
),
|
||||
):
|
||||
"""Print list of protocols"""
|
||||
@@ -112,7 +112,7 @@ def protocol(
|
||||
typer.echo(f"{database_name}.{task_name}.{protocol}")
|
||||
|
||||
|
||||
def duration_to_str(seconds: float) -> Text:
|
||||
def duration_to_str(seconds: float) -> str:
|
||||
hours = math.floor(seconds / 3600)
|
||||
minutes = math.floor((seconds - 3600 * hours) / 60)
|
||||
return f"{hours}h{minutes:02d}m"
|
||||
@@ -137,7 +137,6 @@ def info(protocol: str):
|
||||
typer.Exit(code=1)
|
||||
|
||||
for subset in subsets:
|
||||
|
||||
num_files = 0
|
||||
speakers = set()
|
||||
duration = 0.0
|
||||
@@ -145,8 +144,7 @@ def info(protocol: str):
|
||||
|
||||
def iterate():
|
||||
try:
|
||||
for file in getattr(p, subset)():
|
||||
yield file
|
||||
yield from getattr(p, subset)()
|
||||
except (AttributeError, NotImplementedError):
|
||||
return
|
||||
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2019-2020 CNRS
|
||||
@@ -42,38 +39,29 @@ Protocols:
|
||||
annotated: xxx.uem
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import functools
|
||||
import string
|
||||
|
||||
|
||||
from . import protocol as protocol_module
|
||||
import warnings
|
||||
from collections.abc import Callable
|
||||
from importlib.metadata import entry_points
|
||||
from numbers import Number
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pyannote.database.protocol.protocol import ProtocolFile
|
||||
|
||||
|
||||
import warnings
|
||||
from numbers import Number
|
||||
from typing import Text, Dict, Callable, Any, Union
|
||||
import functools
|
||||
|
||||
from . import protocol as protocol_module
|
||||
from .loader import load_lst, load_trial
|
||||
from .protocol.protocol import Subset
|
||||
from .protocol.segmentation import SegmentationProtocol
|
||||
from .protocol.speaker_diarization import SpeakerDiarizationProtocol
|
||||
|
||||
from importlib.metadata import entry_points
|
||||
|
||||
from .util import get_annotated
|
||||
|
||||
from .loader import load_lst, load_trial
|
||||
|
||||
# All "Loader" classes types (eg RTTMLoader, UEMLoader, ...) retrieved from the entry point.
|
||||
LOADERS = {
|
||||
ep.name: ep
|
||||
for ep in entry_points(group="pyannote.database.loader")
|
||||
}
|
||||
LOADERS = {ep.name: ep for ep in entry_points(group="pyannote.database.loader")}
|
||||
|
||||
|
||||
def Template(template: Text, database_yml: Path) -> Callable[[ProtocolFile], Any]:
|
||||
def Template(template: str, database_yml: Path) -> Callable[[ProtocolFile], Any]:
|
||||
"""Get data loader based on template
|
||||
|
||||
Parameters
|
||||
@@ -83,7 +71,7 @@ def Template(template: Text, database_yml: Path) -> Callable[[ProtocolFile], Any
|
||||
Extension (here ".csv") determined which data loader to use.
|
||||
database_yml : Path
|
||||
Path to YAML configuration file, to which `template` is relative.
|
||||
Defaults to assume that `template` is absolute or relative to
|
||||
Defaults to assume that `template` is absolute or relative to
|
||||
current working directory.
|
||||
|
||||
Returns
|
||||
@@ -120,6 +108,7 @@ def Template(template: Text, database_yml: Path) -> Callable[[ProtocolFile], Any
|
||||
def NumericValue(value):
|
||||
def load(current_file: ProtocolFile):
|
||||
return value
|
||||
|
||||
return load
|
||||
|
||||
|
||||
@@ -129,10 +118,10 @@ def resolve_path(path: Path, database_yml: Path) -> Path:
|
||||
Parameters
|
||||
----------
|
||||
path : `Path`
|
||||
Path. Can be either absolute, relative to current working directory,
|
||||
Path. Can be either absolute, relative to current working directory,
|
||||
or relative to `database_yml` parent directory.
|
||||
database_yml : `Path`
|
||||
Path to YAML configuration file.
|
||||
Path to YAML configuration file.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -155,12 +144,13 @@ def resolve_path(path: Path, database_yml: Path) -> Path:
|
||||
|
||||
|
||||
def meta_subset_iter(
|
||||
meta_database: Text,
|
||||
meta_task: Text,
|
||||
meta_protocol: Text,
|
||||
meta_database: str,
|
||||
meta_task: str,
|
||||
meta_protocol: str,
|
||||
meta_subset: Subset,
|
||||
subset_entries: Dict,
|
||||
subset_entries: dict,
|
||||
database_yml: Path,
|
||||
registry=None,
|
||||
):
|
||||
"""Meta-protocol method that iterates over a subset
|
||||
|
||||
@@ -181,19 +171,19 @@ def meta_subset_iter(
|
||||
REPERE.SpeakerDiarization.Phase2: [train, development]
|
||||
"""
|
||||
|
||||
# this is imported here to avoid circular imports
|
||||
from . import registry
|
||||
# Keep direct callers compatible, but prefer the registry that loaded this protocol.
|
||||
if registry is None:
|
||||
from . import registry
|
||||
|
||||
for protocol, subsets in subset_entries.items():
|
||||
partial_protocol = registry.get_protocol(protocol)
|
||||
for subset in subsets:
|
||||
method_name = f"{subset}_iter"
|
||||
for file in getattr(partial_protocol, method_name)():
|
||||
yield file
|
||||
yield from getattr(partial_protocol, method_name)()
|
||||
|
||||
|
||||
def gather_loaders(
|
||||
entries: Dict,
|
||||
entries: dict,
|
||||
database_yml: Path,
|
||||
) -> dict:
|
||||
"""Loads all Loaders for data type specified in 'entries' into a dict.
|
||||
@@ -215,7 +205,6 @@ def gather_loaders(
|
||||
lazy_loader = dict()
|
||||
|
||||
for key, value in entries.items():
|
||||
|
||||
if key == "uri" or key == "trial":
|
||||
continue
|
||||
|
||||
@@ -228,7 +217,6 @@ def gather_loaders(
|
||||
is_template = len(set(placeholders) - set([None])) > 0
|
||||
|
||||
if is_template:
|
||||
|
||||
# make sure old database.yml specifications still work but warn the user
|
||||
# that they can now get rid of this "_" prefix
|
||||
if value.startswith("_"):
|
||||
@@ -243,7 +231,6 @@ def gather_loaders(
|
||||
lazy_loader[key] = Template(value, database_yml)
|
||||
|
||||
else:
|
||||
|
||||
path = resolve_path(Path(value), database_yml)
|
||||
|
||||
# check if file exists
|
||||
@@ -273,11 +260,11 @@ def gather_loaders(
|
||||
|
||||
def subset_iter(
|
||||
self,
|
||||
database: Text = None,
|
||||
task: Text = None,
|
||||
protocol: Text = None,
|
||||
database: str = None,
|
||||
task: str = None,
|
||||
protocol: str = None,
|
||||
subset: Subset = None,
|
||||
entries: Dict = None,
|
||||
entries: dict = None,
|
||||
database_yml: Path = None,
|
||||
**metadata,
|
||||
):
|
||||
@@ -323,16 +310,18 @@ def subset_iter(
|
||||
|
||||
for uri in uris:
|
||||
yield ProtocolFile(
|
||||
{"uri": uri, "database": database, "subset": subset, **metadata}, lazy=lazy_loader
|
||||
{"uri": uri, "database": database, "subset": subset, **metadata},
|
||||
lazy=lazy_loader,
|
||||
)
|
||||
|
||||
|
||||
def subset_trial(
|
||||
self,
|
||||
database: Text = None,
|
||||
task: Text = None,
|
||||
protocol: Text = None,
|
||||
database: str = None,
|
||||
task: str = None,
|
||||
protocol: str = None,
|
||||
subset: Subset = None,
|
||||
entries: Dict = None,
|
||||
entries: dict = None,
|
||||
database_yml: Path = None,
|
||||
):
|
||||
"""
|
||||
@@ -357,7 +346,7 @@ def subset_trial(
|
||||
lazy_loader["try_with"] = get_annotated
|
||||
|
||||
# meant to store and cache one `ProtocolFile` instance per file
|
||||
files: Dict[Text, ProtocolFile] = dict()
|
||||
files: dict[str, ProtocolFile] = dict()
|
||||
|
||||
# iterate trials and use preloaded test files
|
||||
for trial in load_trial(resolve_path(Path(entries["trial"]), database_yml)):
|
||||
@@ -394,18 +383,19 @@ def get_init(protocols):
|
||||
return init
|
||||
|
||||
|
||||
def get_custom_protocol_class_name(database: Text, task: Text, protocol: Text):
|
||||
def get_custom_protocol_class_name(database: str, task: str, protocol: str):
|
||||
return f"{database}__{task}__{protocol}"
|
||||
|
||||
|
||||
def create_protocol(
|
||||
database: Text,
|
||||
task: Text,
|
||||
protocol: Text,
|
||||
protocol_entries: Dict,
|
||||
database: str,
|
||||
task: str,
|
||||
protocol: str,
|
||||
protocol_entries: dict,
|
||||
database_yml: Path,
|
||||
version: dict | None = None,
|
||||
) -> Union[type, None]:
|
||||
registry=None,
|
||||
) -> type | None:
|
||||
"""Create new protocol class
|
||||
|
||||
Parameters
|
||||
@@ -416,6 +406,8 @@ def create_protocol(
|
||||
protocol_entries : dict
|
||||
version : dict, optional
|
||||
Annotation version as {"name": ..., "metadata": ...}
|
||||
registry : Registry, optional
|
||||
Registry used to resolve members of meta-protocols.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -423,7 +415,6 @@ def create_protocol(
|
||||
|
||||
"""
|
||||
|
||||
|
||||
try:
|
||||
base_class = getattr(
|
||||
protocol_module, "Protocol" if task == "Protocol" else f"{task}Protocol"
|
||||
@@ -466,7 +457,7 @@ def create_protocol(
|
||||
msg = (
|
||||
f"'{database}.{task}.{protocol}' found in {database_yml} does not define "
|
||||
f"the 'scope' of speaker labels (file, database, or global). Setting it to 'file'."
|
||||
)
|
||||
)
|
||||
print(msg)
|
||||
metadata["scope"] = "file"
|
||||
|
||||
@@ -475,7 +466,6 @@ def create_protocol(
|
||||
|
||||
methods = dict()
|
||||
for subset, subset_entries in protocol_entries.items():
|
||||
|
||||
if subset not in [
|
||||
"files",
|
||||
"train",
|
||||
@@ -512,10 +502,10 @@ def create_protocol(
|
||||
subset,
|
||||
subset_entries,
|
||||
database_yml,
|
||||
registry=registry,
|
||||
)
|
||||
)
|
||||
else:
|
||||
|
||||
methods[method_name] = functools.partialmethod(
|
||||
subset_iter,
|
||||
database=database,
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2016 CNRS
|
||||
@@ -27,10 +24,10 @@
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
|
||||
from typing import Optional
|
||||
from .protocol.protocol import Preprocessors
|
||||
import warnings
|
||||
|
||||
from .protocol.protocol import Preprocessors
|
||||
|
||||
|
||||
class Database:
|
||||
"""Base database
|
||||
@@ -49,7 +46,7 @@ class Database:
|
||||
)
|
||||
warnings.warn(msg)
|
||||
|
||||
super(Database, self).__init__()
|
||||
super().__init__()
|
||||
|
||||
def register_protocol(self, task_name, protocol_name, protocol):
|
||||
if not hasattr(self, "protocols_"):
|
||||
@@ -74,9 +71,7 @@ class Database:
|
||||
def get_protocols(self, task):
|
||||
return sorted(self.protocols_[task].keys())
|
||||
|
||||
def get_protocol(
|
||||
self, task, protocol, preprocessors: Optional[Preprocessors] = None
|
||||
):
|
||||
def get_protocol(self, task, protocol, preprocessors: Preprocessors | None = None):
|
||||
return self.protocols_[task][protocol](preprocessors=preprocessors)
|
||||
|
||||
def __str__(self):
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2016- CNRS
|
||||
@@ -31,7 +28,6 @@ import glob
|
||||
import os
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Text
|
||||
|
||||
from pyannote.database.protocol.protocol import ProtocolFile
|
||||
|
||||
@@ -57,12 +53,16 @@ def _scan_candidate(path: Path) -> list[Path]:
|
||||
return [path] if path.is_file() else []
|
||||
|
||||
pattern = glob.escape(str(path)).replace("[*]", "*")
|
||||
return [p for match in glob.iglob(pattern, recursive=True) if (p := Path(match)).is_file()]
|
||||
return [
|
||||
p
|
||||
for match in glob.iglob(pattern, recursive=True)
|
||||
if (p := Path(match)).is_file()
|
||||
]
|
||||
|
||||
|
||||
class FileFinder:
|
||||
"""Database file finder.
|
||||
|
||||
"""Database file finder.
|
||||
|
||||
Retrieve media files by URI.
|
||||
|
||||
Parameters
|
||||
@@ -71,16 +71,15 @@ class FileFinder:
|
||||
Database registry. Defaults to `pyannote.database.registry`.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
registry: Registry = None,
|
||||
database_yml: Text = None):
|
||||
def __init__(self, registry: Registry = None, database_yml: str = None):
|
||||
super().__init__()
|
||||
if registry is None:
|
||||
if database_yml is None:
|
||||
registry = global_registry
|
||||
else:
|
||||
warnings.warn("Passing `database.yml` to `FileFinder` is deprecated in favor of `registry`.")
|
||||
warnings.warn(
|
||||
"Passing `database.yml` to `FileFinder` is deprecated in favor of `registry`."
|
||||
)
|
||||
registry = Registry()
|
||||
registry.load_database(database_yml)
|
||||
self.registry = registry
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2020-2025 CNRS
|
||||
@@ -33,10 +30,11 @@
|
||||
import string
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Any, Text
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
from pyannote.core import Annotation, Timeline
|
||||
|
||||
from pyannote.database.protocol.protocol import ProtocolFile
|
||||
from pyannote.database.util import load_lab, load_rttm, load_uem
|
||||
|
||||
@@ -66,7 +64,7 @@ def load_lst(file_lst):
|
||||
List or uris
|
||||
"""
|
||||
|
||||
with open(file_lst, mode="r") as fp:
|
||||
with open(file_lst) as fp:
|
||||
lines = fp.readlines()
|
||||
return [line.strip() for line in lines]
|
||||
|
||||
@@ -105,7 +103,7 @@ class RTTMLoader:
|
||||
(e.g. "/path/to/{database}/{subset}/{uri}.rttm")
|
||||
"""
|
||||
|
||||
def __init__(self, path: Text = None):
|
||||
def __init__(self, path: str = None):
|
||||
super().__init__()
|
||||
|
||||
self.path = str(path)
|
||||
@@ -156,7 +154,7 @@ class STMLoader:
|
||||
self.placeholders_ = set(placeholders) - set([None])
|
||||
|
||||
if self.placeholders_:
|
||||
self.loaded_: dict[str, "SegLST"] = dict()
|
||||
self.loaded_: dict[str, SegLST] = dict()
|
||||
return
|
||||
|
||||
if MEETEVAL_IS_AVAILABLE:
|
||||
@@ -170,7 +168,7 @@ class STMLoader:
|
||||
return
|
||||
|
||||
warnings.warn("MeetEval is not available, STM files cannot be loaded.")
|
||||
self.loaded_: dict[str, "SegLST"] = dict()
|
||||
self.loaded_: dict[str, SegLST] = dict()
|
||||
|
||||
def __call__(self, file: ProtocolFile) -> "SegLST":
|
||||
uri = file["uri"]
|
||||
@@ -223,7 +221,7 @@ class UEMLoader:
|
||||
(e.g. "/path/to/{database}/{subset}/{uri}.uem")
|
||||
"""
|
||||
|
||||
def __init__(self, path: Text = None):
|
||||
def __init__(self, path: str = None):
|
||||
super().__init__()
|
||||
|
||||
self.path = str(path)
|
||||
@@ -273,7 +271,7 @@ class LABLoader:
|
||||
...
|
||||
"""
|
||||
|
||||
def __init__(self, path: Text = None):
|
||||
def __init__(self, path: str = None):
|
||||
super().__init__()
|
||||
|
||||
self.path = str(path)
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2016- CNRS
|
||||
@@ -26,15 +23,14 @@
|
||||
# AUTHORS
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
from .protocol import Protocol
|
||||
from .collection import CollectionProtocol
|
||||
from .protocol import Protocol
|
||||
from .segmentation import SegmentationProtocol
|
||||
from .speaker_diarization import SpeakerDiarizationProtocol
|
||||
from .speaker_spotting import SpeakerSpottingProtocol
|
||||
from .speaker_verification import SpeakerVerificationProtocol
|
||||
from .speaker_identification import SpeakerIdentificationProtocol
|
||||
from .speaker_recognition import SpeakerRecognitionProtocol
|
||||
|
||||
from .speaker_spotting import SpeakerSpottingProtocol
|
||||
from .speaker_verification import SpeakerVerificationProtocol
|
||||
|
||||
__all__ = [
|
||||
"Protocol",
|
||||
@@ -46,4 +42,3 @@ __all__ = [
|
||||
"SpeakerIdentificationProtocol",
|
||||
"SpeakerRecognitionProtocol",
|
||||
]
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2018-2020 CNRS
|
||||
@@ -27,7 +24,8 @@
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
|
||||
from typing import Iterator, Dict
|
||||
from collections.abc import Iterator
|
||||
|
||||
from .protocol import Protocol
|
||||
|
||||
|
||||
@@ -90,9 +88,9 @@ class CollectionProtocol(Protocol):
|
||||
"""
|
||||
|
||||
# this method should be overriden
|
||||
def files_iter(self) -> Iterator[Dict]:
|
||||
def files_iter(self) -> Iterator[dict]:
|
||||
raise NotImplementedError()
|
||||
|
||||
# this allows Protocol.files() to iterate over the collection
|
||||
def train_iter(self) -> Iterator[Dict]:
|
||||
def train_iter(self) -> Iterator[dict]:
|
||||
return self.files_iter()
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2016-2020 CNRS
|
||||
@@ -33,23 +30,19 @@ Protocols
|
||||
|
||||
"""
|
||||
|
||||
import warnings
|
||||
import collections
|
||||
import threading
|
||||
import itertools
|
||||
from typing import Union, Dict, Iterator, Callable, Any, Text, Optional
|
||||
|
||||
try:
|
||||
from typing import Literal
|
||||
except ImportError:
|
||||
from typing_extensions import Literal
|
||||
import threading
|
||||
import warnings
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Any, Literal, Union
|
||||
|
||||
Subset = Literal["train", "development", "test"]
|
||||
LEGACY_SUBSET_MAPPING = {"train": "trn", "development": "dev", "test": "tst"}
|
||||
Scope = Literal["file", "database", "global"]
|
||||
|
||||
Preprocessor = Callable[["ProtocolFile"], Any]
|
||||
Preprocessors = Dict[Text, Preprocessor]
|
||||
Preprocessors = dict[str, Preprocessor]
|
||||
|
||||
|
||||
class ProtocolFile(collections.abc.MutableMapping):
|
||||
@@ -70,7 +63,7 @@ class ProtocolFile(collections.abc.MutableMapping):
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, precomputed: Union[Dict, "ProtocolFile"], lazy: Dict = None):
|
||||
def __init__(self, precomputed: Union[dict, "ProtocolFile"], lazy: dict = None):
|
||||
|
||||
if lazy is None:
|
||||
lazy = dict()
|
||||
@@ -79,7 +72,7 @@ class ProtocolFile(collections.abc.MutableMapping):
|
||||
# when 'precomputed' is a ProtocolFile, it may already contain lazy keys.
|
||||
|
||||
# we use 'precomputed' precomputed keys as precomputed keys
|
||||
self._store: Dict = abs(precomputed)
|
||||
self._store: dict = abs(precomputed)
|
||||
|
||||
# we handle the corner case where the intersection of 'precomputed' lazy keys
|
||||
# and 'lazy' keys is not empty. this is currently achieved by "unlazying" the
|
||||
@@ -90,7 +83,7 @@ class ProtocolFile(collections.abc.MutableMapping):
|
||||
# we use the union of 'precomputed' lazy keys and provided 'lazy' keys as lazy keys
|
||||
compound_lazy = dict(precomputed.lazy)
|
||||
compound_lazy.update(lazy)
|
||||
self.lazy: Dict = compound_lazy
|
||||
self.lazy: dict = compound_lazy
|
||||
|
||||
else:
|
||||
# when 'precomputed' is a Dict, we use it directly as precomputed keys
|
||||
@@ -124,9 +117,7 @@ class ProtocolFile(collections.abc.MutableMapping):
|
||||
|
||||
def __getitem__(self, key):
|
||||
with self.lock_:
|
||||
|
||||
if key in self.lazy and self.evaluating_[key] == 0:
|
||||
|
||||
# mark lazy key as being evaluated
|
||||
self.evaluating_.update([key])
|
||||
|
||||
@@ -150,7 +141,6 @@ class ProtocolFile(collections.abc.MutableMapping):
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
with self.lock_:
|
||||
|
||||
if key in self.lazy:
|
||||
del self.lazy[key]
|
||||
|
||||
@@ -158,7 +148,6 @@ class ProtocolFile(collections.abc.MutableMapping):
|
||||
|
||||
def __delitem__(self, key):
|
||||
with self.lock_:
|
||||
|
||||
if key in self.lazy:
|
||||
del self.lazy[key]
|
||||
|
||||
@@ -166,7 +155,6 @@ class ProtocolFile(collections.abc.MutableMapping):
|
||||
|
||||
def __iter__(self):
|
||||
with self.lock_:
|
||||
|
||||
store_keys = list(self._store)
|
||||
for key in store_keys:
|
||||
yield key
|
||||
@@ -220,7 +208,6 @@ class ProtocolFile(collections.abc.MutableMapping):
|
||||
|
||||
precomputed = {"uri": uris}
|
||||
for key, value in abs(self).items():
|
||||
|
||||
if key == "uri":
|
||||
continue
|
||||
|
||||
@@ -311,7 +298,7 @@ class Protocol:
|
||||
(e.g. {'audio': '/path/to/{uri}.wav'})
|
||||
"""
|
||||
|
||||
def __init__(self, preprocessors: Optional[Preprocessors] = None):
|
||||
def __init__(self, preprocessors: Preprocessors | None = None):
|
||||
super().__init__()
|
||||
|
||||
if preprocessors is None:
|
||||
@@ -319,7 +306,6 @@ class Protocol:
|
||||
|
||||
self.preprocessors = dict()
|
||||
for key, preprocessor in preprocessors.items():
|
||||
|
||||
if callable(preprocessor):
|
||||
self.preprocessors[key] = preprocessor
|
||||
|
||||
@@ -337,21 +323,21 @@ class Protocol:
|
||||
msg = f'"{key}" preprocessor is neither a callable nor a string.'
|
||||
raise ValueError(msg)
|
||||
|
||||
def preprocess(self, current_file: Union[Dict, ProtocolFile]) -> ProtocolFile:
|
||||
def preprocess(self, current_file: dict | ProtocolFile) -> ProtocolFile:
|
||||
return ProtocolFile(current_file, lazy=self.preprocessors)
|
||||
|
||||
def __str__(self):
|
||||
return self.__doc__
|
||||
|
||||
def train_iter(self) -> Iterator[Union[Dict, ProtocolFile]]:
|
||||
def train_iter(self) -> Iterator[dict | ProtocolFile]:
|
||||
"""Iterate over files in the training subset"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def development_iter(self) -> Iterator[Union[Dict, ProtocolFile]]:
|
||||
def development_iter(self) -> Iterator[dict | ProtocolFile]:
|
||||
"""Iterate over files in the development subset"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def test_iter(self) -> Iterator[Union[Dict, ProtocolFile]]:
|
||||
def test_iter(self) -> Iterator[dict | ProtocolFile]:
|
||||
"""Iterate over files in the test subset"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -402,19 +388,16 @@ class Protocol:
|
||||
"train_enrolment",
|
||||
"train_trial",
|
||||
]:
|
||||
|
||||
if not hasattr(self, method):
|
||||
continue
|
||||
|
||||
def iterate():
|
||||
try:
|
||||
for file in getattr(self, method)():
|
||||
yield file
|
||||
yield from getattr(self, method)()
|
||||
except (AttributeError, NotImplementedError):
|
||||
return
|
||||
|
||||
for current_file in iterate():
|
||||
|
||||
# skip "files" that do not contain a "uri" entry.
|
||||
# this happens for speaker verification trials that contain
|
||||
# two nested files "file1" and "file2"
|
||||
@@ -423,7 +406,6 @@ class Protocol:
|
||||
continue
|
||||
|
||||
for current_file_ in current_file.files():
|
||||
|
||||
# corner case when the same file is yielded several times
|
||||
uri = get_unique_identifier(current_file_)
|
||||
if uri in yielded_uris:
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2016- CNRS
|
||||
@@ -27,20 +24,15 @@
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
|
||||
from typing import Dict, Optional
|
||||
from .protocol import Protocol
|
||||
from .protocol import ProtocolFile
|
||||
from .protocol import Subset
|
||||
from .protocol import Preprocessor
|
||||
from .protocol import Preprocessors
|
||||
from pyannote.core import Annotation
|
||||
from pyannote.core import Timeline
|
||||
from pyannote.core import Segment
|
||||
import functools
|
||||
|
||||
from pyannote.core import Annotation, Segment, Timeline
|
||||
|
||||
from .protocol import Preprocessor, Preprocessors, Protocol, ProtocolFile, Subset
|
||||
|
||||
|
||||
def crop_annotated(
|
||||
current_file: ProtocolFile, existing_preprocessor: Optional[Preprocessor] = None
|
||||
current_file: ProtocolFile, existing_preprocessor: Preprocessor | None = None
|
||||
) -> Timeline:
|
||||
"""Preprocessor that crops 'annotated' according to 'duration'
|
||||
|
||||
@@ -75,14 +67,14 @@ def crop_annotated(
|
||||
# crop 'annotated' to 'duration'
|
||||
duration = Segment(0.0, duration)
|
||||
|
||||
if annotated and not annotated.extent() in duration:
|
||||
if annotated and annotated.extent() not in duration:
|
||||
return annotated.crop(duration, mode="intersection")
|
||||
|
||||
return annotated
|
||||
|
||||
|
||||
def crop_annotation(
|
||||
current_file: ProtocolFile, existing_preprocessor: Optional[Preprocessor] = None
|
||||
current_file: ProtocolFile, existing_preprocessor: Preprocessor | None = None
|
||||
) -> Annotation:
|
||||
"""Preprocessor that crops 'annotation' by 'annotated'
|
||||
|
||||
@@ -215,7 +207,7 @@ class SegmentationProtocol(Protocol):
|
||||
filename2
|
||||
"""
|
||||
|
||||
def __init__(self, preprocessors: Optional[Preprocessors] = None):
|
||||
def __init__(self, preprocessors: Preprocessors | None = None):
|
||||
|
||||
if preprocessors is None:
|
||||
preprocessors = dict()
|
||||
@@ -234,7 +226,7 @@ class SegmentationProtocol(Protocol):
|
||||
|
||||
super().__init__(preprocessors=preprocessors)
|
||||
|
||||
def stats(self, subset: Subset = "train") -> Dict:
|
||||
def stats(self, subset: Subset = "train") -> dict:
|
||||
"""Obtain global statistics on a given subset
|
||||
|
||||
Parameters
|
||||
@@ -263,7 +255,6 @@ class SegmentationProtocol(Protocol):
|
||||
labels = {}
|
||||
|
||||
for item in getattr(self, subset)():
|
||||
|
||||
annotated = get_annotated(item)
|
||||
annotated_duration += annotated.duration()
|
||||
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2016- CNRS
|
||||
@@ -28,6 +25,7 @@
|
||||
|
||||
from .segmentation import SegmentationProtocol
|
||||
|
||||
|
||||
class SpeakerDiarizationProtocol(SegmentationProtocol):
|
||||
"""A protocol for speaker diarization experiments
|
||||
|
||||
@@ -122,4 +120,4 @@ class SpeakerDiarizationProtocol(SegmentationProtocol):
|
||||
filename2
|
||||
"""
|
||||
|
||||
pass
|
||||
pass
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2017 CNRS
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2016-2020 CNRS
|
||||
@@ -44,76 +41,76 @@ class SpeakerRecognitionProtocol(Protocol):
|
||||
|
||||
def trn_iter(self):
|
||||
raise NotImplementedError(
|
||||
"Custom speaker recognition protocol " 'should implement "trn_iter".'
|
||||
'Custom speaker recognition protocol should implement "trn_iter".'
|
||||
)
|
||||
|
||||
def trn_enroll_iter(self):
|
||||
raise NotImplementedError(
|
||||
"Custom speaker recognition protocol " 'should implement "trn_enroll_iter".'
|
||||
'Custom speaker recognition protocol should implement "trn_enroll_iter".'
|
||||
)
|
||||
|
||||
def trn_test_iter(self):
|
||||
raise NotImplementedError(
|
||||
"Custom speaker recognition protocol " 'should implement "trn_test_iter".'
|
||||
'Custom speaker recognition protocol should implement "trn_test_iter".'
|
||||
)
|
||||
|
||||
def trn_keys(self):
|
||||
raise NotImplementedError(
|
||||
"Custom speaker recognition protocol " 'should implement "trn_keys".'
|
||||
'Custom speaker recognition protocol should implement "trn_keys".'
|
||||
)
|
||||
|
||||
def dev_enroll_iter(self):
|
||||
raise NotImplementedError(
|
||||
"Custom speaker recognition protocol " 'should implement "dev_enroll_iter".'
|
||||
'Custom speaker recognition protocol should implement "dev_enroll_iter".'
|
||||
)
|
||||
|
||||
def dev_test_iter(self):
|
||||
raise NotImplementedError(
|
||||
"Custom speaker recognition protocol " 'should implement "dev_test_iter".'
|
||||
'Custom speaker recognition protocol should implement "dev_test_iter".'
|
||||
)
|
||||
|
||||
def dev_keys(self):
|
||||
raise NotImplementedError(
|
||||
"Custom speaker recognition protocol " 'should implement "dev_keys".'
|
||||
'Custom speaker recognition protocol should implement "dev_keys".'
|
||||
)
|
||||
|
||||
def tst_enroll_iter(self):
|
||||
raise NotImplementedError(
|
||||
"Custom speaker recognition protocol " 'should implement "tst_enroll_iter".'
|
||||
'Custom speaker recognition protocol should implement "tst_enroll_iter".'
|
||||
)
|
||||
|
||||
def tst_test_iter(self):
|
||||
raise NotImplementedError(
|
||||
"Custom speaker recognition protocol " 'should implement "tst_test_iter".'
|
||||
'Custom speaker recognition protocol should implement "tst_test_iter".'
|
||||
)
|
||||
|
||||
def tst_keys(self):
|
||||
raise NotImplementedError(
|
||||
"Custom speaker recognition protocol " 'should implement tst_keys".'
|
||||
'Custom speaker recognition protocol should implement tst_keys".'
|
||||
)
|
||||
|
||||
def train(self, yield_name=False):
|
||||
"""Iterate over the training set
|
||||
|
||||
This will yield dictionaries with the followings keys:
|
||||
This will yield dictionaries with the followings keys:
|
||||
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
unique recording identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
* speaker: str
|
||||
unique speaker identifier
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
unique recording identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
* speaker: str
|
||||
unique speaker identifier
|
||||
|
||||
as well as keys coming from the provided preprocessors.
|
||||
as well as keys coming from the provided preprocessors.
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.train():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
... speaker = item['speaker']
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.train():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
... speaker = item['speaker']
|
||||
"""
|
||||
|
||||
generator = self.trn_iter()
|
||||
@@ -127,22 +124,22 @@ Usage
|
||||
def train_enroll(self, yield_name=True):
|
||||
"""Iterate over the training set enrollments
|
||||
|
||||
This will yield dictionaries with the followings keys:
|
||||
This will yield dictionaries with the followings keys:
|
||||
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
|
||||
as well as keys coming from the provided preprocessors.
|
||||
as well as keys coming from the provided preprocessors.
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.train_enroll():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.train_enroll():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
"""
|
||||
|
||||
generator = self.trn_enroll_iter()
|
||||
@@ -156,22 +153,22 @@ Usage
|
||||
def train_test(self, yield_name=True):
|
||||
"""Iterate over the training set tests
|
||||
|
||||
This will yield dictionaries with the followings keys:
|
||||
This will yield dictionaries with the followings keys:
|
||||
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
|
||||
as well as keys coming from the provided preprocessors.
|
||||
as well as keys coming from the provided preprocessors.
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.train_test():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.train_test():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
"""
|
||||
|
||||
generator = self.trn_test_iter()
|
||||
@@ -188,22 +185,22 @@ Usage
|
||||
def development_enroll(self, yield_name=True):
|
||||
"""Iterate over the development set enrollments
|
||||
|
||||
This will yield dictionaries with the followings keys:
|
||||
This will yield dictionaries with the followings keys:
|
||||
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
|
||||
as well as keys coming from the provided preprocessors.
|
||||
as well as keys coming from the provided preprocessors.
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.development_enroll():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.development_enroll():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
"""
|
||||
|
||||
generator = self.dev_enroll_iter()
|
||||
@@ -217,22 +214,22 @@ Usage
|
||||
def development_test(self, yield_name=True):
|
||||
"""Iterate over the development set tests
|
||||
|
||||
This will yield dictionaries with the followings keys:
|
||||
This will yield dictionaries with the followings keys:
|
||||
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
|
||||
as well as keys coming from the provided preprocessors.
|
||||
as well as keys coming from the provided preprocessors.
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.development_test():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.development_test():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
"""
|
||||
|
||||
generator = self.dev_test_iter()
|
||||
@@ -249,25 +246,25 @@ Usage
|
||||
def test_enroll(self, yield_name=True):
|
||||
"""Iterate over the test set targets
|
||||
|
||||
This will yield dictionaries with the followings keys:
|
||||
This will yield dictionaries with the followings keys:
|
||||
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
* speaker: str
|
||||
unique speaker identifier
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
* speaker: str
|
||||
unique speaker identifier
|
||||
|
||||
as well as keys coming from the provided preprocessors.
|
||||
as well as keys coming from the provided preprocessors.
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.test_enroll():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
... speaker = item['speaker']
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.test_enroll():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
... speaker = item['speaker']
|
||||
"""
|
||||
|
||||
generator = self.tst_enroll_iter()
|
||||
@@ -281,22 +278,22 @@ Usage
|
||||
def test_test(self, yield_name=True):
|
||||
"""Iterate over the test set tests
|
||||
|
||||
This will yield dictionaries with the followings keys:
|
||||
This will yield dictionaries with the followings keys:
|
||||
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
* database: str
|
||||
unique database identifier
|
||||
* uri: str
|
||||
uniform (or unique) resource identifier
|
||||
* channel: int
|
||||
index of resource channel to use
|
||||
|
||||
as well as keys coming from the provided preprocessors.
|
||||
as well as keys coming from the provided preprocessors.
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.test_test():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
Usage
|
||||
-----
|
||||
>>> for item in protocol.test_test():
|
||||
... uri = item['uri']
|
||||
... channel = item['channel']
|
||||
"""
|
||||
|
||||
generator = self.tst_test_iter()
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2017-2020 CNRS
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2017-2020 CNRS
|
||||
@@ -27,10 +24,10 @@
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
|
||||
from typing import Dict, Iterator
|
||||
from collections.abc import Iterator
|
||||
|
||||
from .protocol import LEGACY_SUBSET_MAPPING, Subset
|
||||
from .speaker_diarization import SpeakerDiarizationProtocol
|
||||
from .protocol import Subset
|
||||
from .protocol import LEGACY_SUBSET_MAPPING
|
||||
|
||||
|
||||
class SpeakerVerificationProtocol(SpeakerDiarizationProtocol):
|
||||
@@ -134,7 +131,7 @@ class SpeakerVerificationProtocol(SpeakerDiarizationProtocol):
|
||||
As such, they also define regular `{subset}` methods.
|
||||
"""
|
||||
|
||||
def subset_trial_helper(self, subset: Subset) -> Iterator[Dict]:
|
||||
def subset_trial_helper(self, subset: Subset) -> Iterator[dict]:
|
||||
|
||||
try:
|
||||
trials = getattr(self, f"{subset}_trial_iter")()
|
||||
@@ -156,23 +153,23 @@ class SpeakerVerificationProtocol(SpeakerDiarizationProtocol):
|
||||
trial["file2"] = self.preprocess(trial["file2"])
|
||||
yield trial
|
||||
|
||||
def train_trial_iter(self) -> Iterator[Dict]:
|
||||
def train_trial_iter(self) -> Iterator[dict]:
|
||||
"""Iterate over trials in the train subset"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def development_trial_iter(self) -> Iterator[Dict]:
|
||||
def development_trial_iter(self) -> Iterator[dict]:
|
||||
"""Iterate over trials in the development subset"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def test_trial_iter(self) -> Iterator[Dict]:
|
||||
def test_trial_iter(self) -> Iterator[dict]:
|
||||
"""Iterate over trials in the test subset"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def train_trial(self) -> Iterator[Dict]:
|
||||
def train_trial(self) -> Iterator[dict]:
|
||||
return self.subset_trial_helper("train")
|
||||
|
||||
def development_trial(self) -> Iterator[Dict]:
|
||||
def development_trial(self) -> Iterator[dict]:
|
||||
return self.subset_trial_helper("development")
|
||||
|
||||
def test_trial(self) -> Iterator[Dict]:
|
||||
def test_trial(self) -> Iterator[dict]:
|
||||
return self.subset_trial_helper("test")
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2022- CNRS
|
||||
@@ -28,16 +25,17 @@
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
# Alexis PLAQUET
|
||||
|
||||
from enum import Enum
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Set, Optional, Text, Tuple, Type, Union
|
||||
import warnings
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from pyannote.database.protocol.protocol import Preprocessors, Protocol
|
||||
|
||||
from .custom import create_protocol, get_init
|
||||
from .database import Database
|
||||
import yaml
|
||||
|
||||
|
||||
# controls what to do in case of protocol name conflict
|
||||
@@ -47,7 +45,7 @@ class LoadingMode(Enum):
|
||||
ERROR = 2 # raise an error
|
||||
|
||||
|
||||
def _apply_version(entries: Dict, version: Text) -> Dict:
|
||||
def _apply_version(entries: dict, version: str) -> dict:
|
||||
"""Replace {version} placeholder in protocol entries
|
||||
|
||||
Parameters
|
||||
@@ -68,16 +66,18 @@ def _apply_version(entries: Dict, version: Text) -> Dict:
|
||||
return value.replace("{version}", version) if isinstance(value, str) else value
|
||||
|
||||
return {
|
||||
key: {sub(k): sub(v) for k, v in value.items()} if isinstance(value, dict) else value
|
||||
key: {sub(k): sub(v) for k, v in value.items()}
|
||||
if isinstance(value, dict)
|
||||
else value
|
||||
for key, value in entries.items()
|
||||
}
|
||||
|
||||
|
||||
def _expand_versions(
|
||||
db_name: Text,
|
||||
task_name: Text,
|
||||
protocol: Text,
|
||||
protocol_entries: Dict,
|
||||
db_name: str,
|
||||
task_name: str,
|
||||
protocol: str,
|
||||
protocol_entries: dict,
|
||||
database_yml: Path,
|
||||
) -> list[tuple[str, dict, dict | None]]:
|
||||
"""Expand protocol into one protocol per version
|
||||
@@ -125,7 +125,9 @@ def _expand_versions(
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
invalid = [str(n) for n, m in versions.items() if not (m is None or isinstance(m, dict))]
|
||||
invalid = [
|
||||
str(n) for n, m in versions.items() if not (m is None or isinstance(m, dict))
|
||||
]
|
||||
if invalid:
|
||||
msg = f"{where}: metadata of versions {invalid} must be a mapping (e.g. {{closing: 0.2}})."
|
||||
raise ValueError(msg)
|
||||
@@ -143,7 +145,8 @@ def _expand_versions(
|
||||
raise ValueError(msg)
|
||||
|
||||
return [
|
||||
(f"{protocol}@{v['name']}", _apply_version(entries, v["name"]), v) for v in versions
|
||||
(f"{protocol}@{v['name']}", _apply_version(entries, v["name"]), v)
|
||||
for v in versions
|
||||
]
|
||||
|
||||
|
||||
@@ -243,7 +246,7 @@ class Registry:
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
self.configs: Dict[Path, Dict] = dict()
|
||||
self.configs: dict[Path, dict] = dict()
|
||||
|
||||
# Content of the "Database" root item (= where to find file content),
|
||||
# with paths as written in database.yml. Only used by FileFinder.
|
||||
@@ -274,11 +277,11 @@ class Registry:
|
||||
# "DatabaseB": pyannote.database.registry.DatabaseB,
|
||||
# "DatabaseC": pyannote.database.registry.DatabaseC,
|
||||
# "X": pyannote.database.registry.X}
|
||||
self.databases: Dict[Text, Type] = dict()
|
||||
self.databases: dict[str, type] = dict()
|
||||
|
||||
def load_database(
|
||||
self,
|
||||
path: Union[Text, Path],
|
||||
path: str | Path,
|
||||
mode: LoadingMode = LoadingMode.OVERRIDE,
|
||||
):
|
||||
"""Load YAML configuration file into the registry
|
||||
@@ -302,9 +305,9 @@ class Registry:
|
||||
|
||||
def _load_database_helper(
|
||||
self,
|
||||
database_yml: Union[Text, Path],
|
||||
database_yml: str | Path,
|
||||
mode: LoadingMode = LoadingMode.KEEP,
|
||||
loading: Set[Path] = set(),
|
||||
loading: set[Path] = set(),
|
||||
):
|
||||
"""Helper function for recursive loading
|
||||
|
||||
@@ -329,7 +332,7 @@ class Registry:
|
||||
loading.add(database_yml)
|
||||
|
||||
# load configuration
|
||||
with open(database_yml, "r") as f:
|
||||
with open(database_yml) as f:
|
||||
config = yaml.load(f, Loader=yaml.SafeLoader)
|
||||
|
||||
# load every requirement
|
||||
@@ -404,7 +407,7 @@ class Registry:
|
||||
return database(**kwargs)
|
||||
|
||||
def get_protocol(
|
||||
self, name, preprocessors: Optional[Preprocessors] = None
|
||||
self, name, preprocessors: Preprocessors | None = None
|
||||
) -> Protocol:
|
||||
"""Get protocol by full name
|
||||
|
||||
@@ -471,7 +474,7 @@ class Registry:
|
||||
self,
|
||||
db_name,
|
||||
db_entries: dict,
|
||||
database_yml: Union[Text, Path] = None,
|
||||
database_yml: str | Path = None,
|
||||
mode: LoadingMode = LoadingMode.OVERRIDE,
|
||||
):
|
||||
"""Load all protocols from this database into the registry.
|
||||
@@ -489,7 +492,7 @@ class Registry:
|
||||
db_name = str(db_name)
|
||||
|
||||
# maps tuple (task,protocol) to the custom protocol class
|
||||
protocols: Dict[Tuple[Text, Text], Type] = dict()
|
||||
protocols: dict[tuple[str, str], type] = dict()
|
||||
|
||||
for task_name, task_entries in db_entries.items():
|
||||
for protocol, protocol_entries in task_entries.items():
|
||||
@@ -515,6 +518,7 @@ class Registry:
|
||||
entries,
|
||||
database_yml,
|
||||
version=version,
|
||||
registry=self,
|
||||
)
|
||||
if CustomProtocol is None:
|
||||
continue
|
||||
@@ -523,8 +527,13 @@ class Registry:
|
||||
|
||||
# expanded[0] is the default (first listed) version, e.g. "Raw@original":
|
||||
default_protocol, _, default_version = expanded[0]
|
||||
if default_version is not None and (task_name, default_protocol) in protocols:
|
||||
protocols[(task_name, protocol)] = protocols[(task_name, default_protocol)]
|
||||
if (
|
||||
default_version is not None
|
||||
and (task_name, default_protocol) in protocols
|
||||
):
|
||||
protocols[(task_name, protocol)] = protocols[
|
||||
(task_name, default_protocol)
|
||||
]
|
||||
|
||||
# If needed, merge old protocols dict with the new one (according to current override rules)
|
||||
if db_name in self.databases:
|
||||
@@ -558,7 +567,7 @@ class Registry:
|
||||
)
|
||||
|
||||
|
||||
def _env_config_paths() -> List[Path]:
|
||||
def _env_config_paths() -> list[Path]:
|
||||
"""Parse PYANNOTE_DATABASE_CONFIG environment variable
|
||||
|
||||
PYANNOTE_DATABASE_CONFIG may contain multiple paths separation by ";".
|
||||
@@ -579,7 +588,7 @@ def _env_config_paths() -> List[Path]:
|
||||
return paths
|
||||
|
||||
|
||||
def _find_default_ymls() -> List[Path]:
|
||||
def _find_default_ymls() -> list[Path]:
|
||||
"""Get paths to default YAML configuration files
|
||||
|
||||
* $HOME/.pyannote/database.yml
|
||||
@@ -592,7 +601,7 @@ def _find_default_ymls() -> List[Path]:
|
||||
List of existing default YAML configuration files
|
||||
"""
|
||||
|
||||
paths: List[Path] = []
|
||||
paths: list[Path] = []
|
||||
|
||||
home_db_yml = Path("~/.pyannote/database.yml").expanduser()
|
||||
if home_db_yml.is_file():
|
||||
@@ -608,8 +617,8 @@ def _find_default_ymls() -> List[Path]:
|
||||
|
||||
|
||||
def _merge_protocols_inplace(
|
||||
new_protocols: Dict[Tuple[Text, Text], Type],
|
||||
old_protocols: Dict[Tuple[Text, Text], Type],
|
||||
new_protocols: dict[tuple[str, str], type],
|
||||
old_protocols: dict[tuple[str, str], type],
|
||||
mode: LoadingMode,
|
||||
db_name: str,
|
||||
database_yml: str,
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2016-2020 CNRS
|
||||
@@ -26,20 +23,13 @@
|
||||
# AUTHORS
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
import warnings
|
||||
|
||||
import pandas as pd
|
||||
from pyannote.core import Segment, Timeline, Annotation
|
||||
from .protocol.protocol import ProtocolFile
|
||||
from pyannote.core import Annotation, Segment, Timeline
|
||||
|
||||
from typing import Text
|
||||
from typing import Union
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
|
||||
DatabaseName = Text
|
||||
PathTemplate = Text
|
||||
DatabaseName = str
|
||||
PathTemplate = str
|
||||
|
||||
|
||||
def get_unique_identifier(item):
|
||||
@@ -332,7 +322,7 @@ def load_lst(file_lst):
|
||||
List or uris
|
||||
"""
|
||||
|
||||
with open(file_lst, mode="r") as fp:
|
||||
with open(file_lst) as fp:
|
||||
lines = fp.readlines()
|
||||
return [line.strip() for line in lines]
|
||||
|
||||
@@ -351,7 +341,7 @@ def load_mapping(mapping_txt):
|
||||
{1st field: 2nd field} dictionary
|
||||
"""
|
||||
|
||||
with open(mapping_txt, mode="r") as fp:
|
||||
with open(mapping_txt) as fp:
|
||||
lines = fp.readlines()
|
||||
|
||||
mapping = dict()
|
||||
@@ -362,7 +352,7 @@ def load_mapping(mapping_txt):
|
||||
return mapping
|
||||
|
||||
|
||||
class LabelMapper(object):
|
||||
class LabelMapper:
|
||||
"""Label mapper for use as pyannote.database preprocessor
|
||||
|
||||
Parameters
|
||||
|
||||
@@ -10,7 +10,7 @@ Protocols:
|
||||
train:
|
||||
uri: lists/train.lst
|
||||
speaker: rttms/train.rttm
|
||||
transcription: _ctms/{uri}.ctm
|
||||
transcription: stms/{uri}.stm
|
||||
domain: mapping/domain.map
|
||||
|
||||
SpeakerDiarization:
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
filename1 A speaker 0.2 0.5 hello
|
||||
filename1 A speaker 0.3 0.7 world
|
||||
@@ -0,0 +1,3 @@
|
||||
filename2 A speaker 0.2 0.5 how
|
||||
filename2 A speaker 0.5 0.9 are
|
||||
filename2 A speaker 0.9 1.1 you
|
||||
@@ -1,86 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2023- CNRS
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# AUTHORS
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
|
||||
from pyannote.database import registry
|
||||
from pyannote.database.protocol import CollectionProtocol
|
||||
from pyannote.database.protocol import Protocol
|
||||
from pyannote.database.protocol import SpeakerDiarizationProtocol
|
||||
from pyannote.database.protocol import SpeakerVerificationProtocol
|
||||
|
||||
assert "MyDatabase" in registry.databases
|
||||
|
||||
database = registry.get_database("MyDatabase")
|
||||
tasks = database.get_tasks()
|
||||
assert "Collection" in tasks
|
||||
assert "Protocol" in tasks
|
||||
assert "SpeakerDiarization" in tasks
|
||||
assert "SpeakerVerification" in tasks
|
||||
|
||||
assert "MyCollection" in database.get_protocols("Collection")
|
||||
assert "MyProtocol" in database.get_protocols("Protocol")
|
||||
assert "MySpeakerDiarization" in database.get_protocols("SpeakerDiarization")
|
||||
assert "MySpeakerVerification" in database.get_protocols("SpeakerVerification")
|
||||
|
||||
|
||||
collection = registry.get_protocol("MyDatabase.Collection.MyCollection")
|
||||
assert isinstance(collection, CollectionProtocol)
|
||||
|
||||
protocol = registry.get_protocol("MyDatabase.Protocol.MyProtocol")
|
||||
assert isinstance(protocol, Protocol)
|
||||
|
||||
speaker_diarization = registry.get_protocol(
|
||||
"MyDatabase.SpeakerDiarization.MySpeakerDiarization"
|
||||
)
|
||||
assert isinstance(speaker_diarization, SpeakerDiarizationProtocol)
|
||||
|
||||
speaker_verification = registry.get_protocol(
|
||||
"MyDatabase.SpeakerVerification.MySpeakerVerification"
|
||||
)
|
||||
assert isinstance(speaker_verification, SpeakerVerificationProtocol)
|
||||
|
||||
|
||||
files = list(collection.files())
|
||||
assert len(files) == 2
|
||||
|
||||
files = list(protocol.files())
|
||||
assert len(files) == 2
|
||||
|
||||
files = list(speaker_diarization.files())
|
||||
assert len(files) == 2
|
||||
|
||||
files = list(speaker_verification.files())
|
||||
assert len(files) == 2
|
||||
|
||||
|
||||
meta_protocol = registry.get_protocol("X.SpeakerDiarization.MyMetaProtocol")
|
||||
files = list(meta_protocol.train())
|
||||
assert len(files) == 2
|
||||
|
||||
files = list(meta_protocol.development())
|
||||
assert len(files) == 4
|
||||
@@ -0,0 +1,59 @@
|
||||
# Copyright (c) 2025- pyannoteAI
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from importlib.metadata import entry_points
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_cli_help():
|
||||
pytest.importorskip("typer")
|
||||
result = subprocess.run(
|
||||
[str(Path(sys.executable).with_name("pyannote-database")), "--help"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
assert "Usage:" in result.stdout
|
||||
|
||||
|
||||
def test_loader_entry_points():
|
||||
loaders = {
|
||||
ep.name: ep.load() for ep in entry_points(group="pyannote.database.loader")
|
||||
}
|
||||
assert {".rttm", ".uem", ".map", ".lab", ".stm"} <= loaders.keys()
|
||||
assert all(callable(loader) for loader in loaders.values())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("templated", [False, True])
|
||||
def test_stm_loader(tmp_path, templated):
|
||||
pytest.importorskip("meeteval")
|
||||
from meeteval.io.seglst import SegLST
|
||||
|
||||
from pyannote.database.loader import STMLoader
|
||||
|
||||
path = tmp_path / "sample.stm"
|
||||
path.write_text("sample 1 speaker 0.0 1.0 hello world\n")
|
||||
loader = STMLoader(tmp_path / "{uri}.stm" if templated else path)
|
||||
transcript = loader({"uri": "sample"})
|
||||
assert isinstance(transcript, SegLST)
|
||||
assert [segment["words"] for segment in transcript] == ["hello world"]
|
||||
assert list(loader({"uri": "sample"})) == list(transcript)
|
||||
|
||||
|
||||
def test_missing_transcription_extra():
|
||||
code = """
|
||||
import sys
|
||||
import warnings
|
||||
sys.modules['meeteval'] = None
|
||||
from pyannote.database.loader import STMLoader
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter('always')
|
||||
loader = STMLoader('sample.stm')
|
||||
result = loader({'uri': 'sample'})
|
||||
assert result is None
|
||||
assert any('MeetEval' in str(w.message) for w in caught)
|
||||
"""
|
||||
subprocess.run([sys.executable, "-c", code], check=True)
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2023- CNRS
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# AUTHORS
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
|
||||
from pyannote.database.protocol import (
|
||||
CollectionProtocol,
|
||||
Protocol,
|
||||
SpeakerDiarizationProtocol,
|
||||
SpeakerVerificationProtocol,
|
||||
)
|
||||
|
||||
|
||||
def test_configured_protocols():
|
||||
from pathlib import Path
|
||||
|
||||
from pyannote.database.registry import Registry
|
||||
|
||||
registry = Registry()
|
||||
registry.load_database(Path(__file__).parent / "data/database.yml")
|
||||
assert "MyDatabase" in registry.databases
|
||||
|
||||
database = registry.get_database("MyDatabase")
|
||||
tasks = database.get_tasks()
|
||||
assert "Collection" in tasks
|
||||
assert "Protocol" in tasks
|
||||
assert "SpeakerDiarization" in tasks
|
||||
assert "SpeakerVerification" in tasks
|
||||
|
||||
assert "MyCollection" in database.get_protocols("Collection")
|
||||
assert "MyProtocol" in database.get_protocols("Protocol")
|
||||
assert "MySpeakerDiarization" in database.get_protocols("SpeakerDiarization")
|
||||
assert "MySpeakerVerification" in database.get_protocols("SpeakerVerification")
|
||||
|
||||
collection = registry.get_protocol("MyDatabase.Collection.MyCollection")
|
||||
assert isinstance(collection, CollectionProtocol)
|
||||
|
||||
protocol = registry.get_protocol("MyDatabase.Protocol.MyProtocol")
|
||||
assert isinstance(protocol, Protocol)
|
||||
|
||||
speaker_diarization = registry.get_protocol(
|
||||
"MyDatabase.SpeakerDiarization.MySpeakerDiarization"
|
||||
)
|
||||
assert isinstance(speaker_diarization, SpeakerDiarizationProtocol)
|
||||
|
||||
speaker_verification = registry.get_protocol(
|
||||
"MyDatabase.SpeakerVerification.MySpeakerVerification"
|
||||
)
|
||||
assert isinstance(speaker_verification, SpeakerVerificationProtocol)
|
||||
|
||||
files = list(collection.files())
|
||||
assert len(files) == 2
|
||||
|
||||
files = list(protocol.files())
|
||||
assert len(files) == 2
|
||||
|
||||
files = list(speaker_diarization.files())
|
||||
assert len(files) == 2
|
||||
|
||||
files = list(speaker_verification.files())
|
||||
assert len(files) == 2
|
||||
|
||||
meta_protocol = registry.get_protocol("X.SpeakerDiarization.MyMetaProtocol")
|
||||
files = list(meta_protocol.train())
|
||||
assert len(files) == 2
|
||||
|
||||
files = list(meta_protocol.development())
|
||||
assert len(files) == 4
|
||||
+16
-9
@@ -1,5 +1,4 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
@@ -29,10 +28,12 @@
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
|
||||
from pyannote.database.registry import LoadingMode, _merge_protocols_inplace
|
||||
|
||||
|
||||
def test_override_merging_disjoint():
|
||||
protocols1 = {
|
||||
("Task1", "Protocol1"): None,
|
||||
@@ -45,33 +46,39 @@ def test_override_merging_disjoint():
|
||||
warnings.simplefilter("error") # expect no warning
|
||||
_merge_protocols_inplace(protocols1, protocols2, LoadingMode.KEEP, "", "")
|
||||
|
||||
assert ("Task1", "Protocol1",) in protocols1
|
||||
assert ("OtherTask", "Protocol1",) in protocols1
|
||||
assert (
|
||||
"Task1",
|
||||
"Protocol1",
|
||||
) in protocols1
|
||||
assert (
|
||||
"OtherTask",
|
||||
"Protocol1",
|
||||
) in protocols1
|
||||
assert len(protocols1) == 2
|
||||
|
||||
|
||||
def test_override_merging_identical():
|
||||
|
||||
protocols2 = {
|
||||
("Task1", "Protocol1"): None,
|
||||
} # the "old" protocols dict. KEEP override options will keep these entries.
|
||||
} # the "old" protocols dict. KEEP override options will keep these entries.
|
||||
|
||||
# Expect warning and protocols1 to become protocols2 (keep old value)
|
||||
protocols1 = {
|
||||
("Task1", "Protocol1"): 42,
|
||||
}
|
||||
with pytest.warns(Warning) as w:
|
||||
with pytest.warns(Warning):
|
||||
_merge_protocols_inplace(protocols1, protocols2, LoadingMode.KEEP, "", "")
|
||||
assert ("Task1", "Protocol1") in protocols1
|
||||
assert protocols1[("Task1", "Protocol1")] == None
|
||||
assert protocols1[("Task1", "Protocol1")] is None
|
||||
assert len(protocols1) == 1
|
||||
|
||||
|
||||
# Expect warning and protocols1 to keep its value (use new value)
|
||||
protocols1 = {
|
||||
("Task1", "Protocol1"): 42,
|
||||
}
|
||||
with pytest.warns(Warning) as w:
|
||||
with pytest.warns(Warning):
|
||||
_merge_protocols_inplace(protocols1, protocols2, LoadingMode.OVERRIDE, "", "")
|
||||
assert ("Task1", "Protocol1") in protocols1
|
||||
assert protocols1[("Task1", "Protocol1")] == 42
|
||||
assert len(protocols1) == 1
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
@@ -29,13 +28,26 @@
|
||||
# Alexis PLAQUET
|
||||
|
||||
|
||||
from pyannote.database import registry
|
||||
from pathlib import Path
|
||||
|
||||
protocol = registry.get_protocol('MyDatabase.SpeakerVerification.MySpeakerVerification')
|
||||
for elt in protocol.train_trial():
|
||||
print(elt)
|
||||
print(elt['file1']['try_with'])
|
||||
file1_annotation = elt['file1']['annotation']
|
||||
print('annotation : ', file1_annotation)
|
||||
file1_annotated = elt['file1']['annotated']
|
||||
print('annotated : ', file1_annotated)
|
||||
from pyannote.core import Annotation, Timeline
|
||||
|
||||
from pyannote.database.registry import Registry
|
||||
|
||||
|
||||
def test_verification_trials():
|
||||
registry = Registry()
|
||||
registry.load_database(Path(__file__).parent / "data/database.yml")
|
||||
protocol = registry.get_protocol(
|
||||
"MyDatabase.SpeakerVerification.MySpeakerVerification"
|
||||
)
|
||||
trials = list(protocol.train_trial())
|
||||
assert len(trials) == 1
|
||||
trial = trials[0]
|
||||
assert not trial["reference"]
|
||||
for key, uri in (("file1", "filename1"), ("file2", "filename2")):
|
||||
file = trial[key]
|
||||
assert file["uri"] == uri
|
||||
assert isinstance(file["annotation"], Annotation)
|
||||
assert isinstance(file["annotated"], Timeline)
|
||||
assert file["try_with"].duration() > 0
|
||||
+37
-10
@@ -23,7 +23,9 @@ def _duration(file):
|
||||
|
||||
|
||||
def _protocols(database):
|
||||
return set(global_registry.get_database(database).get_protocols("SpeakerDiarization"))
|
||||
return set(
|
||||
global_registry.get_database(database).get_protocols("SpeakerDiarization")
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
@@ -96,8 +98,12 @@ Protocols:
|
||||
def test_selecting_a_version_changes_annotation_only(versioned_dataset):
|
||||
"""Selecting a version changes the annotation content, not which
|
||||
recordings are covered."""
|
||||
original = global_registry.get_protocol("VersionedTestDB.SpeakerDiarization.Raw@original")
|
||||
refined = global_registry.get_protocol("VersionedTestDB.SpeakerDiarization.Raw@refined")
|
||||
original = global_registry.get_protocol(
|
||||
"VersionedTestDB.SpeakerDiarization.Raw@original"
|
||||
)
|
||||
refined = global_registry.get_protocol(
|
||||
"VersionedTestDB.SpeakerDiarization.Raw@refined"
|
||||
)
|
||||
|
||||
assert sorted(f["uri"] for f in original.train()) == ["u1", "u2"]
|
||||
assert sorted(f["uri"] for f in refined.train()) == ["u1", "u2"]
|
||||
@@ -121,7 +127,11 @@ def test_one_protocol_per_version_and_default_is_first_listed(versioned_dataset)
|
||||
("Raw", {"name": "original", "metadata": {}}),
|
||||
("Other", {"name": "refined", "metadata": {"closing": 0.2}}),
|
||||
):
|
||||
files = list(global_registry.get_protocol(f"VersionedTestDB.SpeakerDiarization.{name}").train())
|
||||
files = list(
|
||||
global_registry.get_protocol(
|
||||
f"VersionedTestDB.SpeakerDiarization.{name}"
|
||||
).train()
|
||||
)
|
||||
assert sorted(f["uri"] for f in files) == ["u1", "u2"]
|
||||
assert all(f["version"] == default for f in files)
|
||||
|
||||
@@ -129,9 +139,15 @@ def test_one_protocol_per_version_and_default_is_first_listed(versioned_dataset)
|
||||
def test_versions_without_placeholder_are_still_expanded(versioned_dataset):
|
||||
"""Declaring `versions:` is enough to version a protocol: paths without
|
||||
`{version}` are simply shared by every version."""
|
||||
assert {"Shared", "Shared@original", "Shared@refined"} <= _protocols("VersionedTestDB")
|
||||
assert {"Shared", "Shared@original", "Shared@refined"} <= _protocols(
|
||||
"VersionedTestDB"
|
||||
)
|
||||
|
||||
files = list(global_registry.get_protocol("VersionedTestDB.SpeakerDiarization.Shared@refined").train())
|
||||
files = list(
|
||||
global_registry.get_protocol(
|
||||
"VersionedTestDB.SpeakerDiarization.Shared@refined"
|
||||
).train()
|
||||
)
|
||||
assert all(f["version"]["name"] == "refined" and _duration(f) == 1.0 for f in files)
|
||||
|
||||
|
||||
@@ -150,7 +166,9 @@ def test_unversioned_protocol_stays_flat(versioned_dataset):
|
||||
|
||||
def test_compound_member_with_version(versioned_dataset):
|
||||
"""X meta-protocol members can pin an annotation version with @."""
|
||||
compound = global_registry.get_protocol("X.SpeakerDiarization.VersionedTestCompound")
|
||||
compound = global_registry.get_protocol(
|
||||
"X.SpeakerDiarization.VersionedTestCompound"
|
||||
)
|
||||
files = list(compound.train())
|
||||
|
||||
assert sorted(f["uri"] for f in files) == ["u1", "u2"]
|
||||
@@ -175,14 +193,23 @@ def test_invalid_version_selection_is_rejected(versioned_dataset, name, match):
|
||||
"protocol_block, match",
|
||||
[
|
||||
# "version" is a reserved subset entry, even without `versions:`
|
||||
("Raw:\n train:\n version: not_allowed", r"reserved for annotation versioning"),
|
||||
(
|
||||
"Raw:\n train:\n version: not_allowed",
|
||||
r"reserved for annotation versioning",
|
||||
),
|
||||
# "@" is reserved to select a version
|
||||
("Foo@Bar:\n scope: file", r"Foo@Bar.*must not contain '@'"),
|
||||
# versions map names to (optional) metadata mappings
|
||||
("Raw:\n versions: [original, refined]", r"must be a non-empty mapping"),
|
||||
("Raw:\n versions: {}", r"must be a non-empty mapping"),
|
||||
("Raw:\n versions: {original, refined: 0.2}", r"metadata of versions \['refined'\]"),
|
||||
("Raw:\n versions: {original, refined: 0}", r"metadata of versions \['refined'\]"),
|
||||
(
|
||||
"Raw:\n versions: {original, refined: 0.2}",
|
||||
r"metadata of versions \['refined'\]",
|
||||
),
|
||||
(
|
||||
"Raw:\n versions: {original, refined: 0}",
|
||||
r"metadata of versions \['refined'\]",
|
||||
),
|
||||
# "." would make the protocol class unpicklable
|
||||
("Raw:\n versions: {original, v2.1}", r"must not contain '\.'.*v2\.1"),
|
||||
],
|
||||
|
||||
Reference in new issue
Block a user