Align database CI and releases; require Python 3.11 (#122)

* Align database CI and releases; fix meta-protocol registry lookup

* Refactor import of Literal in protocol.py

Removed the import error handling for Literal and directly imported it from typing.

* Use uv add for transcription installation in README

* Remove and ignore uv.lock
This commit is contained in:
Hervé BREDIN authored and GitHub committed 2026-11-02 17:19:40 +01:00
1 parent 37df51de71
commit d5b07685d6
45 files changed
+1305 -2609

No files matched your search

Binary file not shown.

After

Width:  |  Height:  |  Size: 702 KiB

+35
View File
@@ -0,0 +1,35 @@
name: Build Wheels
on:
workflow_call:
inputs:
release-tag:
description: Release tag; empty on pull requests
required: false
type: string
default: ""
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# hatch-vcs needs Git history and tags to derive the package version.
fetch-depth: 0
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version-file: .python-version
- name: Build wheel and source distribution
run: uv build
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: python-package-distributions
path: dist/
if-no-files-found: error
retention-days: ${{ inputs.release-tag != '' && 90 || 3 }}
+87
View File
@@ -0,0 +1,87 @@
name: CI
on:
pull_request:
release:
types: [published]
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# lint (PR only)
# build ──┬──► test ──────────┬──► publish (release only)
# └──► version-guard ─┘
# ci-complete collects all required results.
jobs:
lint:
if: github.event_name == 'pull_request'
uses: ./.github/workflows/lint.yml
build:
uses: ./.github/workflows/build-wheels.yml
with:
release-tag: ${{ github.event.release.tag_name || '' }}
test:
needs: build
uses: ./.github/workflows/test.yml
version-guard:
name: versions
if: github.event_name == 'release'
needs: build
uses: ./.github/workflows/version-guard.yml
with:
release-tag: ${{ github.event.release.tag_name }}
# PyPI trusted publishing must run in a top-level workflow.
publish:
if: github.event_name == 'release'
needs: [build, test, version-guard]
runs-on: ubuntu-latest
environment:
name: pypi
url: https://pypi.org/project/pyannote-database/
permissions:
contents: write
id-token: write
steps:
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: python-package-distributions
path: dist/
- name: Upload distributions to GitHub Release
uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2
with:
tag_name: ${{ github.event.release.tag_name }}
files: |
dist/*.whl
dist/*.tar.gz
fail_on_unmatched_files: true
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- name: Upload to PyPI
run: >-
uv publish --trusted-publishing always
--publish-url https://upload.pypi.org/legacy/
--check-url https://pypi.org/simple/
ci-complete:
if: always()
needs: [lint, build, test, version-guard, publish]
runs-on: ubuntu-latest
env:
RESULTS: ${{ join(needs.*.result, ' ') }}
steps:
- name: Check results
run: |
for result in $RESULTS; do
if [[ "$result" == failure || "$result" == cancelled ]]; then
echo "::error::Required job finished with result: $result"
exit 1
fi
done
+47 -25
View File
@@ -1,33 +1,55 @@
name: Documentation
on:
pull_request:
push:
branches:
- master
branches: [main]
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
build-and-deploy:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
fetch-depth: 0
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
cache-dependency-glob: uv.lock
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# hatch-vcs needs Git history and tags to derive the package version.
fetch-depth: 0
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version-file: .python-version
- name: Install documentation dependencies
run: |
uv venv
uv pip install '.[doc,cli]'
- name: Build documentation
run: uv run --no-project --python .venv/bin/python sphinx-build -W --keep-going -b html doc/source doc/build/html
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: documentation
path: doc/build/html
include-hidden-files: true
retention-days: 3
- name: Install the project
run: uv sync --extra doc
- name: Build documentation
run: |
make --directory=doc html
touch ./doc/build/html/.nojekyll
- name: Deploy
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./doc/build/html
deploy:
if: github.event_name == 'push'
needs: build
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: documentation
path: doc/build/html
- uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: doc/build/html
+30
View File
@@ -0,0 +1,30 @@
name: Lint
on:
workflow_call:
permissions:
contents: read
jobs:
ruff:
name: python
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# Report lint/format issues without blocking PRs.
# Remove continue-on-error once the codebase is clean.
- name: Run ruff check
uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b # v4.0.0
continue-on-error: true
with:
version: "0.17.1"
args: "check ."
- name: Run ruff format
uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b # v4.0.0
continue-on-error: true
with:
version: "0.17.1"
args: "format --check ."
-125
View File
@@ -1,125 +0,0 @@
name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
on: push
jobs:
build:
name: Build distribution 📦
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
fetch-depth: 0
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
cache-dependency-glob: uv.lock
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version-file: ".python-version"
- name: Build
run: uv build
- name: Store the distribution packages
uses: actions/upload-artifact@v4
with:
name: python-package-distributions
path: dist/
publish-to-pypi:
name: >-
Publish Python 🐍 distribution 📦 to PyPI
if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
needs:
- build
runs-on: ubuntu-latest
environment:
name: pypi
permissions:
id-token: write
steps:
- name: Download all the dists
uses: actions/download-artifact@v4
with:
name: python-package-distributions
path: dist/
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
cache-dependency-glob: uv.lock
- name: Publish distribution 📦 to PyPI
run: uv publish --trusted-publishing always --publish-url https://upload.pypi.org/legacy/
github-release:
name: >-
Sign the Python 🐍 distribution 📦 with Sigstore
and upload them to GitHub Release
needs:
- publish-to-pypi
runs-on: ubuntu-latest
permissions:
contents: write # IMPORTANT: mandatory for making GitHub Releases
id-token: write # IMPORTANT: mandatory for sigstore
steps:
- name: Download all the dists
uses: actions/download-artifact@v4
with:
name: python-package-distributions
path: dist/
- name: Sign the dists with Sigstore
uses: sigstore/gh-action-sigstore-python@v3.0.0
with:
inputs: >-
./dist/*.tar.gz
./dist/*.whl
- name: Create GitHub Release
env:
GITHUB_TOKEN: ${{ github.token }}
run: >-
gh release create
"$GITHUB_REF_NAME"
--repo "$GITHUB_REPOSITORY"
--notes ""
- name: Upload artifact signatures to GitHub Release
env:
GITHUB_TOKEN: ${{ github.token }}
# Upload to GitHub Release using the `gh` CLI.
# `dist/` contains the built packages, and the
# sigstore-produced signatures and certificates.
run: >-
gh release upload
"$GITHUB_REF_NAME" dist/**
--repo "$GITHUB_REPOSITORY"
# publish-to-testpypi:
# name: Publish Python 🐍 distribution 📦 to TestPyPI
# needs:
# - build
# runs-on: ubuntu-latest
#
# environment:
# name: testpypi
#
# permissions:
# id-token: write # IMPORTANT: mandatory for trusted publishing
#
# steps:
# - name: Download all the dists
# uses: actions/download-artifact@v4
# with:
# name: python-package-distributions
# path: dist/
# - name: Install uv
# uses: astral-sh/setup-uv@v5
# with:
# enable-cache: true
# cache-dependency-glob: uv.lock
# - name: Publish distribution 📦 to PyPI
# run: uv publish --trusted-publishing always --publish-url https://test.pypi.org/legacy/
+47 -24
View File
@@ -1,36 +1,59 @@
name: Test
on:
pull_request:
branches:
- develop
push:
branches:
- develop
- master
- release/*
workflow_call:
permissions:
contents: read
jobs:
test:
name: Test
name: Test Python ${{ matrix.python-version }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version:
- "3.10"
- "3.11"
- "3.12"
env:
UV_PYTHON: ${{ matrix.python-version }}
python-version: ["3.11", "3.12", "3.13", "3.14", "3.15", "3.x"]
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- id: python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: ${{ matrix.python-version }}
check-latest: ${{ matrix.python-version == '3.x' }}
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: python-package-distributions
path: dist/
- name: Install the built wheel with test dependencies
run: |
uv venv --python "${{ steps.python.outputs.python-path }}"
uv pip install "$(echo dist/*.whl)[test]"
- name: Run tests against the installed wheel
run: uv run --no-project --python .venv/bin/python python -m pytest tests
- name: Install uv
uses: astral-sh/setup-uv@v5
- name: Install the project
run: uv sync --extra test
- name: Run tests
run: uv run pytest tests
optional-features:
name: Optional features (Python 3.11)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- id: python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.11"
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: python-package-distributions
path: dist/
- name: Install the built wheel with test dependencies
run: |
uv venv --python "${{ steps.python.outputs.python-path }}"
uv pip install "$(echo dist/*.whl)[test,cli,transcription]"
- name: Run tests against the installed wheel
run: uv run --no-project --python .venv/bin/python python -m pytest tests/test_optional_features.py
+70
View File
@@ -0,0 +1,70 @@
name: Version Guard
on:
workflow_call:
inputs:
release-tag:
description: Release tag expected to match the package version
required: true
type: string
permissions:
contents: read
jobs:
not-already-published:
name: pyannote-database
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version-file: .python-version
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: python-package-distributions
path: dist/
- name: Compare distributions with PyPI
env:
RELEASE_TAG: ${{ inputs.release-tag }}
shell: python
run: |
import hashlib
import json
import os
from pathlib import Path
from urllib.error import HTTPError
from urllib.request import urlopen
from zipfile import ZipFile
from email.parser import BytesParser
directory = Path("dist")
tag = os.environ["RELEASE_TAG"]
wheels = list(directory.glob("*.whl"))
sources = list(directory.glob("*.tar.gz"))
if len(wheels) != 1 or len(sources) != 1:
raise ValueError("Expected exactly one wheel and one source distribution")
with ZipFile(wheels[0]) as wheel:
metadata_path, = [n for n in wheel.namelist() if n.endswith(".dist-info/METADATA")]
metadata = BytesParser().parsebytes(wheel.read(metadata_path))
version = metadata["Version"]
if metadata["Name"] != "pyannote-database" or version != tag.removeprefix("v"):
raise ValueError(f"Release tag {tag!r} does not match pyannote-database {version}")
try:
with urlopen(f"https://pypi.org/pypi/pyannote-database/{version}/json", timeout=30) as response:
published = json.load(response)["urls"]
except HTTPError as error:
if error.code != 404:
raise
published = []
hashes = {item["filename"]: item["digests"]["sha256"] for item in published}
for artifact in wheels + sources:
if artifact.name not in hashes:
print(f"{artifact.name}: not published yet")
elif hashlib.sha256(artifact.read_bytes()).hexdigest() == hashes[artifact.name]:
print(f"{artifact.name}: already published with identical contents")
else:
raise ValueError(f"{artifact.name} already exists with different contents. Use a new version.")
+7
View File
@@ -89,3 +89,10 @@ ENV/
.ropeproject
.mypy_cache
.venv/
.ruff_cache/
.pytest_cache/
# Local dependency resolution
uv.lock
+7
View File
@@ -0,0 +1,7 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.17.1
hooks:
- id: ruff-check
args: [--fix]
- id: ruff-format
+1 -1
View File
@@ -1 +1 @@
3.10
3.11
+297
View File
@@ -0,0 +1,297 @@
# Changelog
## Next version
- BREAKING: require Python 3.11 or newer; Python 3.10 is no longer supported
- fix: resolve meta-protocol members using the registry that loaded them
- BREAKING: forbid `@` in protocol names (now used to select a version)
- BREAKING: reserve `version` key in subsets (now used to store the annotation version)
- fix: fix `X` meta-protocols with Python 3.14
- fix: `FileFinder` patterns only match files (not directories nor hidden files)
- feat: add annotation versions with optional metadata (selected with `Database.Task.Protocol@version`)
- feat: add `PYANNOTE_DATABASE_DATA_DIRS` environment variable to set root directories of relative `Databases:` paths
## Version 6.1.1 (2025-12-07)
- fix: fix warning about invalid espace sequence
## Version 6.1.0 (2025-09-19)
- BREAKING: update `STMLoader` to return `meeteval.io.SegLST` instances
## Version 6.0.0 (2025-09-09)
- BREAKING: drop support to `Python` < 3.10
- BREAKING: switch to native namespace package
- setup: switch to `uv`
## Version 5.1.3 (2025-01-15)
- chore: remove deprecated use of `delim_whitespace`
- chore: use `importlib.metadata` instead of `pkg_resources`
## Version 5.1.0 (2024-04-05)
- BREAKING: switch to python 3.9+ (no support for 3.12 yet)
- feat(registry): make registry iterable (yields protocol names)
- fix(setup): fix `typer`` dependency
## Version 5.0.1 (2023-04-21)
- fix: fix support for numeric values in custom protocols (e.g. `channel: 0`)
- improve: do not complain about missing scope for meta protocols
## Version 5.0.0 (2023-03-27)
- feat(registry): add pyannote.database.registry to load multiple YAML configuration file
- BREAKING: turn pyannote.database.get_{database|protocol|protocols} function into registry methods
- BREAKING: remove support for database plugins (via "pyannote.database.database" entrypoint)
- feat(protocol): add generic segmentation protocol with "classes" metadata
- feat(protocol): add support for label "scope" in speaker diarization protocols
- improve(protocol): make ProtocolFile pickable
- feat(loader): add support for LAB and STM files
- feat(loader): add option to filter RTTM lines by "type"
- fix(cli): fix corner case when annotation key is missing
## Version 4.1.3 (2022-02-11)
- fix: fix corner case with missing uris in RTTM and UEM files
- fix(CI): fix documentation deployment
## Version 4.1.1 (2021-08-4)
- fix: fix support for meta verification protocol
## Version 4.1 (2021-04-07)
- feat: make {RTTM|UEM}Loader usable as preprocessors
- improve: remove the need for underscore prefix in database.yml
- fix: fix ProtocolFile.__iter__ (dictionary changed size during iteration)
## Version 4.0.4 (2020-12-14)
- fix: fix ProtocolFile.__iter__ (dictionary changed size during iteration)
## Version 4.0.3 (2020-10-11)
- fix: make custom protocols pickable
## Version 4.0.2 (2020-10-02)
- feat: automagically crop "annotated" to file "duration" when available
- fix: fix a bug in SpeakerDiarizationProtocols
- improve: avoid loading ProtocolFile (lazy) keys in "get_unique_identifier"
## Version 4.0.1 (2020-06-26)
- feat: set "name" attribute in get_protocol
- fix: display warning only when precomputed value is modified
## Version 4.0 (2020-06-15)
- feat: add support for custom speaker verification protocols
- feat: add pyannote.database.loader entrypoint
- feat: add pyannote-database CLI
- feat: add a few dataloaders (RTTM, UEM, CTM, MAP)
- feat: add support for nested ProtocolFile
- doc: major documentation update (README and docstrings)
- BREAKING: custom protocols must define a "uri" section
- BREAKING: remove support for "preprocessors" in Database constructor
- BREAKING: remove support for progress bars
## Version 3.0.1 (2020-03-31)
- BREAKING (feat): if a "database.yml" file exists in current working directory, it will be used even if PYANNOTE_DATABASE_CONFIG is set to another value.
- feat: add support in FileFinder for paths relative to "database.yml"
- BREAKING: rename "config_yml" option to "database_yml" in FileFinder
- feat: add support in custom protocols for paths relative to "database.yml" (@PaulLerner)
- BREAKING (feat): use "annotated" to crop "annotation" in custom protocols (@PaulLerner)
- fix: add support for int-like protocol name in custom protocols (@PaulLerner)
## Version 2.5 (2020-02-04)
- BREAKING: refactor {current | protocol}_file_iter
- BREAKING: only rely on "uri" to decide if a ProtocolFile contains multiple files
- BREAKING: deprecate FileFinder.current_file_iter in favor of ProtocolFile.files
- BREAKING: deprecate FileFinder.protocol_file_iter in favor of Protocol.files
- fix: fix support for lazy preprocessors in {Protocol | ProtocolFile}.files
## Version 2.4.3 (2020-01-24)
- fix: fix infinite recursion in "ProtocolFile" lazy evaluation
## Version 2.4.2 (2020-01-06)
- fix: add support for int-like database name in custom protocol
## Version 2.4.1 (2019-12-20)
- fix: make ProtocolFile thread-safe
## Version 2.4 (2019-12-17)
- feat: make preprocessors lazy
- fix: pandas would convert a label to NaN ([@PaulLerner](https://github.com/PaulLerner))
- feat: setup continuous integration
- setup: switch to pyannote.core 3.2
## Version 2.3.1 (2019-09-04)
- fix: fix support for MDTM files in `pyannote.database.custom.subset_iter` ([#23](https://github.com/pyannote/pyannote-database/issues/23))
## Version 2.3 (2019-07-19)
- feat: add LabelMapper preprocessor ([@MarvinLvn](https://github.com/MarvinLvn))
- chore: replace (deprecated) pandas.read_table with pandas.read_csv ([@V-assim](https://github.com/V-assim))
- chore: use YAML safe loader ([@V-assim](https://github.com/V-assim))
## Version 2.2 (2019-06-26)
- setup: switch to pyannote.core 3.0
- feat: add RTTMLoader preprocessor
## Version 2.1 (2019-04-04)
- feat: add support for "domain" key in generic protocol
## Version 2.0 (2019-03-20)
- BREAKING: change location and format of pyannote.database configuration file
- feat: add support for PYANNOTE_DATABASE_CONFIG environment variable
## Version 1.6 (2019-03-12)
- feat: add support for [file-based speaker diarization protocols](https://github.com/pyannote/pyannote-database/tree/develop#generic-speaker-diarization-protocols)
- setup: switch to pyannote.core 2.1
## Version 1.5.5 (2018-11-30)
- fix: fix Collection.files when progress=True
## Version 1.5.4 (2018-11-14)
- fix: skip files with no "uri" entry in FileFinder.protocol_file_iter
## Version 1.5.3 (2018-11-08)
- fix: fix broken SpeakerVerificationProtocol
## Version 1.5.1 (2018-10-16)
- fix: fix support for string preprocessors
## Version 1.5 (2018-09-25)
- BREAKING: simplify SpeakerVerificationProtocol with {subset}_trial methods
## Version 1.4 (2018-07-13)
- feat: add raw collection protocol
## Version 1.3.2 (2018-05-16)
- fix: fix regression introduced in 1.3.1
## Version 1.3.1 (2018-05-11)
- fix: fix bug in `FileFinder.protocol_file_iter` with empty iterators
## Version 1.3 (2018-02-04)
- feat: add `extra_keys` parameter to `{protocol | current}_file_iter`
## Version 1.2.1 (2018-02-03)
- setup: drop support for Python 2
- feat: add `protocol_file_iter` and `current_file_iter` to FileFinder
- feat: add `get_label_identifier` utility function
- fix: fix "get_unique_identifier" when "database" or "channel" is None
## Version 1.1 (2017-10-13)
- feat: add speaker identification protocol
- feat: add speaker verification protocols
- feat: add support for list of uris in FileFinder
## Version 1.0 (2017-10-02)
- feat: add support for "meta" protocols
- feat: add speaker spotting protocol
- setup: switch to pyannote.core 1.1
## Version 0.12 (2017-06-28)
- feat: add utility functions at package root
- doc: improve documentation
- doc: add link to pyannote-db-template repository
## Version 0.11.2 (2017-03-15)
- fix: fix a bug with string template preprocessors
- doc: improve documentation
## Version 0.11.1 (2017-01_16)
- feat: add 'get_protocol' helper function
## Version 0.11 (2017-01-11)
- feat: add support for validation on training set to speaker recognition protocols
- feat: add 'get_annotated' helper function
## Version 0.10.2 (2017-01-04)
- fix: fix bug in FileFinder
## Version 0.10.1 (2016-12-17)
- improve: change signature of preprocessor.__call__
## Version 0.9 (2016-12-14)
- feat: add "get_unique_identifier" utility function
## Version 0.8.1 (2016-12-12)
- fix: fix progress bar support
## Version 0.8 (2016-12-06)
- feat: add progress bar support
## Version 0.7.1 (2016-12-03)
- fix: add 'yield_name' parameter to speaker recognition generators
## Version 0.7 (2016-12-02)
- feat: add speaker recognition protocol
## Version 0.6.1 (2016-12-02)
- feat: add FileFinder utility class
- fix: fix SpeakerDiarizationProtocol.stats()
## Version 0.5 (2016-12-01)
- BREAKING: replace 'medium_template' by (more generic) 'preprocessors'
## Version 0.4.1 (2016-11-17)
- fix: rename 'speakers' to 'labels' in statistics dictionary
## Version 0.4 (2016-10-27)
- feat: add a method providing global statistics about a subset
## Version 0.3 (2016-09-22)
- feat: add support for multiple media
## Version 0.2 (2016-09-21)
- feat: add support for 'medium_template' attribute
## Version 0.1 (2016-09-20)
- first public version
+104 -69
View File
@@ -1,3 +1,5 @@
![pyannote.database social preview](https://raw.githubusercontent.com/pyannote/pyannote-database/main/.github/social-preview.png)
# pyannote-database
Reproducible experimental protocols for multimedia (audio, video, text) database.
@@ -75,6 +77,7 @@ Since version `5.0`, configuration files must be loaded into the `registry` like
```python
from pyannote.database import registry
registry.load_database("/path/to/database.yml")
```
@@ -93,9 +96,10 @@ Once loaded in the registry, protocols can be used in Python like this:
```python
from pyannote.database import registry
registry.load_database("/path/to/database.yml")
protocol = registry.get_protocol('MyDatabase.Protocol.MyProtocol')
protocol = registry.get_protocol("MyDatabase.Protocol.MyProtocol")
for resource in protocol.train():
print(resource["uri"])
filename1
@@ -125,6 +129,8 @@ with the content of `database.yml` as follows:
## Data loaders
STM transcription loading requires the optional dependencies: `uv add "pyannote.database[transcription]"`.
The above `MyDatabase.Protocol.MyProtocol` protocol is not very useful as it only allows to iterate over a list of resources with a single `'uri'` key. Metadata can be added to each resource with the following syntax:
```yaml
@@ -135,7 +141,7 @@ The above `MyDatabase.Protocol.MyProtocol` protocol is not very useful as it onl
train:
uri: lists/train.lst
speaker: rttms/train.rttm
transcription: ctms/{uri}.ctm
transcription: stms/{uri}.stm
```
and the following directory structure:
@@ -147,60 +153,61 @@ and the following directory structure:
| └── train.lst
├── rttms
| └── train.rttm
└── ctms
├── filename1.ctm
└── filename2.ctm
└── stms
├── filename1.stm
└── filename2.stm
```
Now, resources have both `'speaker'` and `'transcription'` keys:
```python
from pyannote.database import registry
protocol = registry.get_protocol('MyDatabase.Protocol.MyProtocol')
protocol = registry.get_protocol("MyDatabase.Protocol.MyProtocol")
for resource in protocol.train():
assert "speaker" in resource
assert isinstance(resource["speaker"], pyannote.core.Annotation)
assert "transcription" in resource
assert isinstance(resource["transcription"], spacy.tokens.Doc)
assert isinstance(resource["transcription"], meeteval.io.SegLST)
```
What happened exactly? Data loaders were automatically selected based on metadata file suffix:
- `pyannote.database.loader.RTTMLoader` for `speaker` entry with `.rttm` suffix
- `pyannote.database.loader.CTMLoader` for `transcription` entry with `ctm` suffix).
- `pyannote.database.loader.STMLoader` for `transcription` entry with `.stm` suffix).
and used to populate `speaker` and `transcription` keys. In pseudo-code:
```python
# instantiate loader registered with `.rttm` suffix
speaker = RTTMLoader('rttms/train.rttm')
speaker = RTTMLoader("rttms/train.rttm")
# entries with {placeholders} serve as path templates
transcription_template = 'ctms/{uri}.ctm'
transcription_template = "stms/{uri}.stm"
for resource in protocol.train():
# unique resource identifier
uri = resource['uri']
uri = resource["uri"]
# only select parts of `rttms/train.rttm` that are relevant to current resource,
# convert it into a convenient data structure (here pyannote.core.Annotation),
# and assign it to `'speaker'` resource key
resource['speaker'] = speaker[uri]
# convert it into a convenient data structure (here pyannote.core.Annotation),
# and assign it to `'speaker'` resource key
resource["speaker"] = speaker(resource)
# replace placeholders in `transcription` path template
ctm = transcription_template.format(uri=uri)
stm = transcription_template.format(uri=uri)
# instantiate loader registered with `.ctm` suffix
transcription = CTMLoader(ctm)
# instantiate loader registered with `.stm` suffix
transcription = STMLoader(stm)
# only select parts of the `ctms/{uri}.ctm` that are relevant to current resource
# only select parts of the `stms/{uri}.stm` that are relevant to current resource
# (here, most likely the whole file), convert it into a convenient data structure
# (here spacy.tokens.Doc), and assign it to `'transcription'` resource key
resource['transcription'] = transcription[uri]
# (here meeteval.io.SegLST), and assign it to `'transcription'` resource key
resource["transcription"] = transcription(resource)
```
`pyannote.database` provides built-in data loaders for a limited set of file formats: `RTTMLoader` for `.rttm` files, `UEMLoader` for `.uem` files, and `CTMLoader` for `.ctm` files. See [Custom data loaders](#custom-data-loaders) section to learn how to add your own.
`pyannote.database` provides built-in data loaders for a limited set of file formats: `RTTMLoader` for `.rttm` files, `UEMLoader` for `.uem` files, and `STMLoader` for `.stm` files. See [Custom data loaders](#custom-data-loaders) section to learn how to add your own.
## Preprocessors
@@ -313,10 +320,11 @@ It can the be used in Python like this:
```python
from pyannote.database import registry
collection = registry.get_protocol('MyDatabase.Collection.MyCollection')
collection = registry.get_protocol("MyDatabase.Collection.MyCollection")
for file in collection.files():
print(file["uri"])
print(file["uri"])
filename1
filename2
filename3
@@ -379,12 +387,13 @@ It can then be used in Python like this:
```python
from pyannote.database import registry
protocol = registry.get_protocol('MyDatabase.Segmentation.MyProtocol')
protocol = registry.get_protocol("MyDatabase.Segmentation.MyProtocol")
for file in protocol.train():
print(file["uri"])
assert "annotation" in file
assert "annotated" in file
print(file["uri"])
assert "annotation" in file
assert "annotated" in file
filename1
filename2
```
@@ -443,12 +452,13 @@ It can then be used in Python like this:
```python
from pyannote.database import registry
protocol = registry.get_protocol('MyDatabase.SpeakerDiarization.MyProtocol')
protocol = registry.get_protocol("MyDatabase.SpeakerDiarization.MyProtocol")
for file in protocol.train():
print(file["uri"])
assert "annotation" in file
assert "annotated" in file
print(file["uri"])
assert "annotation" in file
assert "annotated" in file
filename1
filename2
```
@@ -546,11 +556,16 @@ Select a version by appending `@version` to the protocol name:
```python
from pyannote.database import registry
protocol = registry.get_protocol('MyDatabase.SpeakerDiarization.MyProtocol@forced_alignment')
protocol = registry.get_protocol(
"MyDatabase.SpeakerDiarization.MyProtocol@forced_alignment"
)
for file in protocol.train():
file["annotation"] # loaded from /path/to/forced_alignment/train.rttm
file["version"] # {"name": "forced_alignment", "metadata": {"closing": 0.2, "aligner": "mfa"}}
file[
"version"
] # {"name": "forced_alignment", "metadata": {"closing": 0.2, "aligner": "mfa"}}
```
Without `@version`, the first version listed (here `original`) is used.
@@ -589,8 +604,9 @@ This new "meta-protocol" can be used like any other protocol of the (fake) `X` d
```python
from pyannote.database import registry
protocol = registry.get_protocol('X.Protocol.MyMetaProtocol')
protocol = registry.get_protocol("X.Protocol.MyMetaProtocol")
for resource in protocol.train():
pass
```
@@ -628,7 +644,7 @@ One can get a list of tasks using `get_tasks` method:
### Custom data loaders
`pyannote.database` provides built-in data loaders for a limited set of file formats: `RTTMLoader` for `.rttm` files, `UEMLoader` for `.uem` files, and `CTMLoader` for `.ctm` files.
`pyannote.database` provides built-in data loaders for a limited set of file formats: `RTTMLoader` for `.rttm` files, `UEMLoader` for `.uem` files, and `STMLoader` for `.stm` files.
In case those are not enough, `pyannote.database` supports the addition of custom data loaders using the `pyannote.database.loader` entry point.
@@ -641,33 +657,37 @@ Here is an example of a Python package called `your_package` that defines two cu
from pyannote.database import ProtocolFile
from pathlib import Path
class Ext1Loader:
def __init__(self, ext1: Path):
print(f'Initializing Ext1Loader with {ext1}')
print(f"Initializing Ext1Loader with {ext1}")
# your code should obviously do something smarter.
# see pyannote.database.loader.RTTMLoader for an example.
self.ext1 = ext1
def __call__(self, current_file: ProtocolFile) -> Text:
uri = current_file["uri"]
print(f'Processing {uri} with Ext1Loader')
print(f"Processing {uri} with Ext1Loader")
# your code should obviously do something smarter.
# see pyannote.database.loader.RTTMLoader for an example.
return f'{uri}.ext1'
return f"{uri}.ext1"
class Ext2Loader:
def __init__(self, ext2: Path):
print(f'Initializing Ext2Loader with {ext2}')
print(f"Initializing Ext2Loader with {ext2}")
# your code should obviously do something smarter.
# see pyannote.database.loader.RTTMLoader for an example.
self.ext2 = ext2
def __call__(self, current_file: ProtocolFile) -> Text:
uri = current_file["uri"]
print(f'Processing {uri} with Ext2Loader')
print(f"Processing {uri} with Ext2Loader")
# your code should obviously do something smarter.
# see pyannote.database.loader.RTTMLoader for an example.
return f'{uri}.ext2'
return f"{uri}.ext2"
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
```
@@ -796,7 +816,7 @@ It can then be used in Python like this:
```python
collection = MyCollection()
for file in collection.files():
print(file["uri"])
print(file["uri"])
filename1
filename2
filename3
@@ -811,12 +831,16 @@ least one of `train_iter`, `development_iter` and `test_iter` methods:
```python
class MySpeakerDiarizationProtocol(SpeakerDiarizationProtocol):
def train_iter(self) -> Iterator[Dict]:
yield {"uri": "filename1",
"annotation": Annotation(...),
"annotated": Timeline(...)}
yield {"uri": "filename2",
"annotation": Annotation(...),
"annotated": Timeline(...)}
yield {
"uri": "filename1",
"annotation": Annotation(...),
"annotated": Timeline(...),
}
yield {
"uri": "filename2",
"annotation": Annotation(...),
"annotated": Timeline(...),
}
```
`{subset}_iter` should return an iterator of dictionnaries with
@@ -840,7 +864,7 @@ It can then be used in Python like this:
```python
protocol = MySpeakerDiarizationProtocol()
for file in protocol.train():
print(file["uri"])
print(file["uri"])
filename1
filename2
```
@@ -854,26 +878,24 @@ A speaker verification protocol can be defined programmatically by creating a cl
```python
class MySpeakerVerificationProtocol(SpeakerVerificationProtocol):
def train_iter(self) -> Iterator[Dict]:
yield {"uri": "filename1",
"annotation": Annotation(...),
"annotated": Timeline(...)}
yield {"uri": "filename2",
"annotation": Annotation(...),
"annotated": Timeline(...)}
yield {
"uri": "filename1",
"annotation": Annotation(...),
"annotated": Timeline(...),
}
yield {
"uri": "filename2",
"annotation": Annotation(...),
"annotated": Timeline(...),
}
def train_trial_iter(self) -> Iterator[Dict]:
yield {"reference": 1,
"file1": ProtocolFile(...),
"file2": ProtocolFile(...)}
yield {"reference": 0,
"file1": {
"uri":"filename1",
"try_with":Timeline(...)
},
"file1": {
"uri":"filename3",
"try_with":Timeline(...)
}
}
yield {"reference": 1, "file1": ProtocolFile(...), "file2": ProtocolFile(...)}
yield {
"reference": 0,
"file1": {"uri": "filename1", "try_with": Timeline(...)},
"file1": {"uri": "filename3", "try_with": Timeline(...)},
}
```
`{subset}_trial_iter` should return an iterator of dictionnaries with
@@ -897,3 +919,16 @@ It can then be used in Python like this:
1 filename1 filename2
0 filename1 filename3
```
## Development
Requires Python 3.11 or newer.
```bash
uv sync --extra test --extra cli --extra transcription
uv run pytest tests
uvx ruff check .
uvx ruff format --check .
```
See the [changelog](https://github.com/pyannote/pyannote-database/blob/main/CHANGELOG.md) for release history.
-355
View File
@@ -1,355 +0,0 @@
#########
Changelog
#########
Develop
~~~~~~~
- BREAKING: forbid `@` in protocol names (now used to select a version)
- BREAKING: reserve `version` key in subsets (now used to store the annotation version)
- fix: fix `X` meta-protocols with Python 3.14
- fix: `FileFinder` patterns only match files (not directories nor hidden files)
- feat: add annotation versions with optional metadata (selected with `Database.Task.Protocol@version`)
- feat: add `PYANNOTE_DATABASE_DATA_DIRS` environment variable to set root directories of relative `Databases:` paths
Version 6.1.1 (2025-12-07)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix warning about invalid espace sequence
Version 6.1.0 (2025-09-19)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- BREAKING: update `STMLoader` to return `meeteval.io.SegLST` instances
Version 6.0.0 (2025-09-09)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- BREAKING: drop support to `Python` < 3.10
- BREAKING: switch to native namespace package
- setup: switch to `uv`
Version 5.1.3 (2025-01-15)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- chore: remove deprecated use of `delim_whitespace`
- chore: use `importlib.metadata` instead of `pkg_resources`
Version 5.1.0 (2024-04-05)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- BREAKING: switch to python 3.9+ (no support for 3.12 yet)
- feat(registry): make registry iterable (yields protocol names)
- fix(setup): fix `typer`` dependency
Version 5.0.1 (2023-04-21)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix support for numeric values in custom protocols (e.g. `channel: 0`)
- improve: do not complain about missing scope for meta protocols
Version 5.0.0 (2023-03-27)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- feat(registry): add pyannote.database.registry to load multiple YAML configuration file
- BREAKING: turn pyannote.database.get_{database|protocol|protocols} function into registry methods
- BREAKING: remove support for database plugins (via "pyannote.database.database" entrypoint)
- feat(protocol): add generic segmentation protocol with "classes" metadata
- feat(protocol): add support for label "scope" in speaker diarization protocols
- improve(protocol): make ProtocolFile pickable
- feat(loader): add support for LAB and STM files
- feat(loader): add option to filter RTTM lines by "type"
- fix(cli): fix corner case when annotation key is missing
Version 4.1.3 (2022-02-11)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix corner case with missing uris in RTTM and UEM files
- fix(CI): fix documentation deployment
Version 4.1.1 (2021-08-4)
~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix support for meta verification protocol
Version 4.1 (2021-04-07)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: make {RTTM|UEM}Loader usable as preprocessors
- improve: remove the need for underscore prefix in database.yml
- fix: fix ProtocolFile.__iter__ (dictionary changed size during iteration)
Version 4.0.4 (2020-12-14)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix ProtocolFile.__iter__ (dictionary changed size during iteration)
Version 4.0.3 (2020-10-11)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: make custom protocols pickable
Version 4.0.2 (2020-10-02)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- feat: automagically crop "annotated" to file "duration" when available
- fix: fix a bug in SpeakerDiarizationProtocols
- improve: avoid loading ProtocolFile (lazy) keys in "get_unique_identifier"
Version 4.0.1 (2020-06-26)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- feat: set "name" attribute in get_protocol
- fix: display warning only when precomputed value is modified
Version 4.0 (2020-06-15)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add support for custom speaker verification protocols
- feat: add pyannote.database.loader entrypoint
- feat: add pyannote-database CLI
- feat: add a few dataloaders (RTTM, UEM, CTM, MAP)
- feat: add support for nested ProtocolFile
- doc: major documentation update (README and docstrings)
- BREAKING: custom protocols must define a "uri" section
- BREAKING: remove support for "preprocessors" in Database constructor
- BREAKING: remove support for progress bars
Version 3.0.1 (2020-03-31)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- BREAKING (feat): if a "database.yml" file exists in current working directory, it will be used even if PYANNOTE_DATABASE_CONFIG is set to another value.
- feat: add support in FileFinder for paths relative to "database.yml"
- BREAKING: rename "config_yml" option to "database_yml" in FileFinder
- feat: add support in custom protocols for paths relative to "database.yml" (@PaulLerner)
- BREAKING (feat): use "annotated" to crop "annotation" in custom protocols (@PaulLerner)
- fix: add support for int-like protocol name in custom protocols (@PaulLerner)
Version 2.5 (2020-02-04)
~~~~~~~~~~~~~~~~~~~~~~~~
- BREAKING: refactor {current | protocol}_file_iter
- BREAKING: only rely on "uri" to decide if a ProtocolFile contains multiple files
- BREAKING: deprecate FileFinder.current_file_iter in favor of ProtocolFile.files
- BREAKING: deprecate FileFinder.protocol_file_iter in favor of Protocol.files
- fix: fix support for lazy preprocessors in {Protocol | ProtocolFile}.files
Version 2.4.3 (2020-01-24)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix infinite recursion in "ProtocolFile" lazy evaluation
Version 2.4.2 (2020-01-06)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: add support for int-like database name in custom protocol
Version 2.4.1 (2019-12-20)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: make ProtocolFile thread-safe
Version 2.4 (2019-12-17)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: make preprocessors lazy
- fix: pandas would convert a label to NaN ([@PaulLerner](https://github.com/PaulLerner))
- feat: setup continuous integration
- setup: switch to pyannote.core 3.2
Version 2.3.1 (2019-09-04)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix support for MDTM files in `pyannote.database.custom.subset_iter` ([#23](https://github.com/pyannote/pyannote-database/issues/23))
Version 2.3 (2019-07-19)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add LabelMapper preprocessor ([@MarvinLvn](https://github.com/MarvinLvn))
- chore: replace (deprecated) pandas.read_table with pandas.read_csv ([@V-assim](https://github.com/V-assim))
- chore: use YAML safe loader ([@V-assim](https://github.com/V-assim))
Version 2.2 (2019-06-26)
~~~~~~~~~~~~~~~~~~~~~~~~
- setup: switch to pyannote.core 3.0
- feat: add RTTMLoader preprocessor
Version 2.1 (2019-04-04)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add support for "domain" key in generic protocol
Version 2.0 (2019-03-20)
~~~~~~~~~~~~~~~~~~~~~~~~
- BREAKING: change location and format of pyannote.database configuration file
- feat: add support for PYANNOTE_DATABASE_CONFIG environment variable
Version 1.6 (2019-03-12)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add support for [file-based speaker diarization protocols](https://github.com/pyannote/pyannote-database/tree/develop#generic-speaker-diarization-protocols)
- setup: switch to pyannote.core 2.1
Version 1.5.5 (2018-11-30)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix Collection.files when progress=True
Version 1.5.4 (2018-11-14)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: skip files with no "uri" entry in FileFinder.protocol_file_iter
Version 1.5.3 (2018-11-08)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix broken SpeakerVerificationProtocol
Version 1.5.1 (2018-10-16)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix support for string preprocessors
Version 1.5 (2018-09-25)
~~~~~~~~~~~~~~~~~~~~~~~~
- BREAKING: simplify SpeakerVerificationProtocol with {subset}_trial methods
Version 1.4 (2018-07-13)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add raw collection protocol
Version 1.3.2 (2018-05-16)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix regression introduced in 1.3.1
Version 1.3.1 (2018-05-11)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix bug in `FileFinder.protocol_file_iter` with empty iterators
Version 1.3 (2018-02-04)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add `extra_keys` parameter to `{protocol | current}_file_iter`
Version 1.2.1 (2018-02-03)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- setup: drop support for Python 2
- feat: add `protocol_file_iter` and `current_file_iter` to FileFinder
- feat: add `get_label_identifier` utility function
- fix: fix "get_unique_identifier" when "database" or "channel" is None
Version 1.1 (2017-10-13)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add speaker identification protocol
- feat: add speaker verification protocols
- feat: add support for list of uris in FileFinder
Version 1.0 (2017-10-02)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add support for "meta" protocols
- feat: add speaker spotting protocol
- setup: switch to pyannote.core 1.1
Version 0.12 (2017-06-28)
~~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add utility functions at package root
- doc: improve documentation
- doc: add link to pyannote-db-template repository
Version 0.11.2 (2017-03-15)
~~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix a bug with string template preprocessors
- doc: improve documentation
Version 0.11.1 (2017-01_16)
~~~~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add 'get_protocol' helper function
Version 0.11 (2017-01-11)
~~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add support for validation on training set to speaker recognition protocols
- feat: add 'get_annotated' helper function
Version 0.10.2 (2017-01-04)
~~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix bug in FileFinder
Version 0.10.1 (2016-12-17)
~~~~~~~~~~~~~~~~~~~~~~~~~~~
- improve: change signature of preprocessor.__call__
Version 0.9 (2016-12-14)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add "get_unique_identifier" utility function
Version 0.8.1 (2016-12-12)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: fix progress bar support
Version 0.8 (2016-12-06)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add progress bar support
Version 0.7.1 (2016-12-03)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: add 'yield_name' parameter to speaker recognition generators
Version 0.7 (2016-12-02)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add speaker recognition protocol
Version 0.6.1 (2016-12-02)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add FileFinder utility class
- fix: fix SpeakerDiarizationProtocol.stats()
Version 0.5 (2016-12-01)
~~~~~~~~~~~~~~~~~~~~~~~~
- BREAKING: replace 'medium_template' by (more generic) 'preprocessors'
Version 0.4.1 (2016-11-17)
~~~~~~~~~~~~~~~~~~~~~~~~~~
- fix: rename 'speakers' to 'labels' in statistics dictionary
Version 0.4 (2016-10-27)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add a method providing global statistics about a subset
Version 0.3 (2016-09-22)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add support for multiple media
Version 0.2 (2016-09-21)
~~~~~~~~~~~~~~~~~~~~~~~~
- feat: add support for 'medium_template' attribute
Version 0.1 (2016-09-20)
~~~~~~~~~~~~~~~~~~~~~~~~
- first public version
+42 -38
View File
@@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
#
# pyannote.database documentation build configuration file, created by
# sphinx-quickstart on Tue Jan 24 15:45:55 2017.
@@ -18,10 +17,7 @@
#
# allow pyannote.database import
import os
import sys
sys.path.insert(0, os.path.abspath('../..'))
from importlib.metadata import version as package_version
# -- General configuration ------------------------------------------------
@@ -32,15 +28,16 @@ sys.path.insert(0, os.path.abspath('../..'))
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.autodoc',
'sphinx.ext.napoleon',
'sphinx.ext.intersphinx',
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.mathjax',
'sphinx.ext.viewcode',
'sphinx.ext.githubpages',
]
extensions = [
"sphinx.ext.autodoc",
"sphinx.ext.napoleon",
"sphinx.ext.intersphinx",
"sphinx.ext.todo",
"sphinx.ext.coverage",
"sphinx.ext.mathjax",
"sphinx.ext.viewcode",
"sphinx.ext.githubpages",
]
# Napoleon settings
@@ -58,31 +55,31 @@ napoleon_use_rtype = True
napoleon_use_keyword = True
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = {'.rst': 'restructuredtext'}
source_suffix = {".rst": "restructuredtext"}
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = u'pyannote.database'
copyright = u'2019, CNRS'
author = u'Hervé Bredin'
project = "pyannote.database"
copyright = "2019, CNRS"
author = "Hervé Bredin"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
import pyannote.database
# The short X.Y version.
version = pyannote.database.__version__.split('+')[0]
version = package_version("pyannote-database").split("+")[0]
# The full version, including alpha/beta/rc tags.
release = pyannote.database.__version__
release = package_version("pyannote-database")
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
@@ -97,7 +94,7 @@ language = "en"
exclude_patterns = []
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True
@@ -119,13 +116,13 @@ html_theme = "sphinx_rtd_theme"
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_static_path = []
# -- Options for HTMLHelp output ------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'pyannotedatabasedoc'
htmlhelp_basename = "pyannotedatabasedoc"
# -- Options for LaTeX output ---------------------------------------------
@@ -134,15 +131,12 @@ latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
@@ -152,8 +146,13 @@ latex_elements = {
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'pyannotedatabase.tex', u'pyannote.database Documentation',
u'Hervé Bredin', 'manual'),
(
master_doc,
"pyannotedatabase.tex",
"pyannote.database Documentation",
"Hervé Bredin",
"manual",
),
]
@@ -162,8 +161,7 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'pyannotedatabase', u'pyannote.database Documentation',
[author], 1)
(master_doc, "pyannotedatabase", "pyannote.database Documentation", [author], 1)
]
@@ -173,12 +171,18 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'pyannotedatabase', u'pyannote.database Documentation',
author, 'pyannotedatabase', 'One line description of project.',
'Miscellaneous'),
(
master_doc,
"pyannotedatabase",
"pyannote.database Documentation",
author,
"pyannotedatabase",
"One line description of project.",
"Miscellaneous",
),
]
intersphinx_mapping = {
'python': ('https://docs.python.org/', None),
'pyannote.core': ('https://pyannote.github.io/pyannote-core', None),
"python": ("https://docs.python.org/3/", None),
"pyannote.core": ("https://pyannote.github.io/pyannote-core", None),
}
+2 -2
View File
@@ -14,7 +14,7 @@ Installation
::
$ pip install pyannote.database
pip install pyannote.database
API documentation
@@ -23,4 +23,4 @@ API documentation
.. toctree::
:maxdepth: 2
changelog
Changelog <https://github.com/pyannote/pyannote-database/blob/main/CHANGELOG.md>
+8 -2
View File
@@ -5,7 +5,7 @@ readme = "README.md"
authors = [
{ name = "Hervé BREDIN", email = "herve@pyannote.ai" }
]
requires-python = ">=3.10"
requires-python = ">=3.11"
dynamic = [
"version",
@@ -43,7 +43,7 @@ transcription = [
]
[build-system]
requires = ["hatchling", "hatch-vcs"]
requires = ["hatchling==1.32.0", "hatch-vcs"]
build-backend = "hatchling.build"
[tool.hatch.version]
@@ -56,3 +56,9 @@ packages = ["src/pyannote"]
dev = [
"ipykernel>=6.29.5",
]
[tool.ruff]
target-version = "py311"
[tool.ruff.lint]
select = ["E4", "E7", "E9", "F", "I", "UP"]
-84
View File
@@ -1,84 +0,0 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2016-2020 CNRS
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr
import versioneer
from setuptools import setup, find_packages
setup(
# package
namespace_packages=["pyannote"],
packages=find_packages(),
install_requires=[
"pyannote.core >= 4.1",
"pyYAML >= 3.12",
"pandas >= 0.19",
"typer >= 0.12.1",
"typing_extensions >= 3.7.4;python_version < '3.8'",
],
entry_points={
"console_scripts": [
"pyannote-database=pyannote.database.cli:main",
],
"pyannote.database.loader": [
".rttm = pyannote.database.loader:RTTMLoader",
".uem = pyannote.database.loader:UEMLoader",
".ctm = pyannote.database.loader:CTMLoader",
".map = pyannote.database.loader:MAPLoader",
".lab = pyannote.database.loader:LABLoader",
".stm = pyannote.database.loader:STMLoader",
],
},
# versioneer
version=versioneer.get_version(),
cmdclass=versioneer.get_cmdclass(),
# PyPI
name="pyannote.database",
description=("Interface to multimedia databases and experimental protocols"),
author="Hervé Bredin",
author_email="bredin@limsi.fr",
url="http://pyannote.github.io/",
classifiers=[
"Development Status :: 4 - Beta",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: MIT License",
"Natural Language :: English",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Topic :: Scientific/Engineering",
],
extras_require={
"testing": ["pytest", "flake8==3.7.9"],
"doc": [
"matplotlib >= 2.0.0",
"Sphinx == 2.2.2",
"ipython == 7.16.3",
"sphinx_rtd_theme == 0.4.3",
],
},
)
+8 -20
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2016- CNRS
@@ -29,29 +26,19 @@
"""pyannote.database"""
from typing import Optional
import importlib.metadata
import warnings
from .registry import registry, LoadingMode
from .database import Database
from .protocol.protocol import Protocol
from .protocol.protocol import ProtocolFile
from .protocol.protocol import Subset
from .protocol.protocol import Preprocessors
from .file_finder import FileFinder
from .util import get_annotated
from .util import get_unique_identifier
from .util import get_label_identifier
from .protocol.protocol import Preprocessors, Protocol, ProtocolFile, Subset
from .registry import LoadingMode, registry
from .util import get_annotated, get_label_identifier, get_unique_identifier
import importlib.metadata
__version__ = importlib.metadata.version("pyannote-database")
def get_protocol(name, preprocessors: Optional[Preprocessors] = None) -> Protocol:
def get_protocol(name, preprocessors: Preprocessors | None = None) -> Protocol:
"""Get protocol by full name
name : str
@@ -68,8 +55,9 @@ def get_protocol(name, preprocessors: Optional[Preprocessors] = None) -> Protoco
Protocol instance
"""
warnings.warn(
"`get_protocol` has been deprecated in favor of `pyannote.database.registry.get_protocol`.",
DeprecationWarning)
"`get_protocol` has been deprecated in favor of `pyannote.database.registry.get_protocol`.",
DeprecationWarning,
)
return registry.get_protocol(name, preprocessors=preprocessors)
+15 -17
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2020- CNRS
@@ -28,20 +25,19 @@
# Alexis PLAQUET
import typer
from enum import Enum
import math
from typing import Text
from pyannote.database import Database
from pyannote.database import registry
from pyannote.database.protocol import CollectionProtocol
from pyannote.database.protocol import SpeakerDiarizationProtocol
from enum import StrEnum
import typer
from pyannote.core import Annotation
from pyannote.database import Database, registry
from pyannote.database.protocol import CollectionProtocol, SpeakerDiarizationProtocol
app = typer.Typer()
class Task(str, Enum):
class Task(StrEnum):
Any = "Any"
Protocol = "Protocol"
Collection = "Collection"
@@ -65,7 +61,7 @@ def task(
metavar="DATABASE",
help="Filter tasks by DATABASE.",
case_sensitive=False,
)
),
):
"""Print list of tasks"""
@@ -90,7 +86,11 @@ def protocol(
case_sensitive=False,
),
task: Task = typer.Option(
"Any", "--task", "-t", help="Filter protocols by TASK.", case_sensitive=False,
"Any",
"--task",
"-t",
help="Filter protocols by TASK.",
case_sensitive=False,
),
):
"""Print list of protocols"""
@@ -112,7 +112,7 @@ def protocol(
typer.echo(f"{database_name}.{task_name}.{protocol}")
def duration_to_str(seconds: float) -> Text:
def duration_to_str(seconds: float) -> str:
hours = math.floor(seconds / 3600)
minutes = math.floor((seconds - 3600 * hours) / 60)
return f"{hours}h{minutes:02d}m"
@@ -137,7 +137,6 @@ def info(protocol: str):
typer.Exit(code=1)
for subset in subsets:
num_files = 0
speakers = set()
duration = 0.0
@@ -145,8 +144,7 @@ def info(protocol: str):
def iterate():
try:
for file in getattr(p, subset)():
yield file
yield from getattr(p, subset)()
except (AttributeError, NotImplementedError):
return
+48 -58
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2019-2020 CNRS
@@ -42,38 +39,29 @@ Protocols:
annotated: xxx.uem
"""
from pathlib import Path
import functools
import string
from . import protocol as protocol_module
import warnings
from collections.abc import Callable
from importlib.metadata import entry_points
from numbers import Number
from pathlib import Path
from typing import Any
from pyannote.database.protocol.protocol import ProtocolFile
import warnings
from numbers import Number
from typing import Text, Dict, Callable, Any, Union
import functools
from . import protocol as protocol_module
from .loader import load_lst, load_trial
from .protocol.protocol import Subset
from .protocol.segmentation import SegmentationProtocol
from .protocol.speaker_diarization import SpeakerDiarizationProtocol
from importlib.metadata import entry_points
from .util import get_annotated
from .loader import load_lst, load_trial
# All "Loader" classes types (eg RTTMLoader, UEMLoader, ...) retrieved from the entry point.
LOADERS = {
ep.name: ep
for ep in entry_points(group="pyannote.database.loader")
}
LOADERS = {ep.name: ep for ep in entry_points(group="pyannote.database.loader")}
def Template(template: Text, database_yml: Path) -> Callable[[ProtocolFile], Any]:
def Template(template: str, database_yml: Path) -> Callable[[ProtocolFile], Any]:
"""Get data loader based on template
Parameters
@@ -83,7 +71,7 @@ def Template(template: Text, database_yml: Path) -> Callable[[ProtocolFile], Any
Extension (here ".csv") determined which data loader to use.
database_yml : Path
Path to YAML configuration file, to which `template` is relative.
Defaults to assume that `template` is absolute or relative to
Defaults to assume that `template` is absolute or relative to
current working directory.
Returns
@@ -120,6 +108,7 @@ def Template(template: Text, database_yml: Path) -> Callable[[ProtocolFile], Any
def NumericValue(value):
def load(current_file: ProtocolFile):
return value
return load
@@ -129,10 +118,10 @@ def resolve_path(path: Path, database_yml: Path) -> Path:
Parameters
----------
path : `Path`
Path. Can be either absolute, relative to current working directory,
Path. Can be either absolute, relative to current working directory,
or relative to `database_yml` parent directory.
database_yml : `Path`
Path to YAML configuration file.
Path to YAML configuration file.
Returns
-------
@@ -155,12 +144,13 @@ def resolve_path(path: Path, database_yml: Path) -> Path:
def meta_subset_iter(
meta_database: Text,
meta_task: Text,
meta_protocol: Text,
meta_database: str,
meta_task: str,
meta_protocol: str,
meta_subset: Subset,
subset_entries: Dict,
subset_entries: dict,
database_yml: Path,
registry=None,
):
"""Meta-protocol method that iterates over a subset
@@ -181,19 +171,19 @@ def meta_subset_iter(
REPERE.SpeakerDiarization.Phase2: [train, development]
"""
# this is imported here to avoid circular imports
from . import registry
# Keep direct callers compatible, but prefer the registry that loaded this protocol.
if registry is None:
from . import registry
for protocol, subsets in subset_entries.items():
partial_protocol = registry.get_protocol(protocol)
for subset in subsets:
method_name = f"{subset}_iter"
for file in getattr(partial_protocol, method_name)():
yield file
yield from getattr(partial_protocol, method_name)()
def gather_loaders(
entries: Dict,
entries: dict,
database_yml: Path,
) -> dict:
"""Loads all Loaders for data type specified in 'entries' into a dict.
@@ -215,7 +205,6 @@ def gather_loaders(
lazy_loader = dict()
for key, value in entries.items():
if key == "uri" or key == "trial":
continue
@@ -228,7 +217,6 @@ def gather_loaders(
is_template = len(set(placeholders) - set([None])) > 0
if is_template:
# make sure old database.yml specifications still work but warn the user
# that they can now get rid of this "_" prefix
if value.startswith("_"):
@@ -243,7 +231,6 @@ def gather_loaders(
lazy_loader[key] = Template(value, database_yml)
else:
path = resolve_path(Path(value), database_yml)
# check if file exists
@@ -273,11 +260,11 @@ def gather_loaders(
def subset_iter(
self,
database: Text = None,
task: Text = None,
protocol: Text = None,
database: str = None,
task: str = None,
protocol: str = None,
subset: Subset = None,
entries: Dict = None,
entries: dict = None,
database_yml: Path = None,
**metadata,
):
@@ -323,16 +310,18 @@ def subset_iter(
for uri in uris:
yield ProtocolFile(
{"uri": uri, "database": database, "subset": subset, **metadata}, lazy=lazy_loader
{"uri": uri, "database": database, "subset": subset, **metadata},
lazy=lazy_loader,
)
def subset_trial(
self,
database: Text = None,
task: Text = None,
protocol: Text = None,
database: str = None,
task: str = None,
protocol: str = None,
subset: Subset = None,
entries: Dict = None,
entries: dict = None,
database_yml: Path = None,
):
"""
@@ -357,7 +346,7 @@ def subset_trial(
lazy_loader["try_with"] = get_annotated
# meant to store and cache one `ProtocolFile` instance per file
files: Dict[Text, ProtocolFile] = dict()
files: dict[str, ProtocolFile] = dict()
# iterate trials and use preloaded test files
for trial in load_trial(resolve_path(Path(entries["trial"]), database_yml)):
@@ -394,18 +383,19 @@ def get_init(protocols):
return init
def get_custom_protocol_class_name(database: Text, task: Text, protocol: Text):
def get_custom_protocol_class_name(database: str, task: str, protocol: str):
return f"{database}__{task}__{protocol}"
def create_protocol(
database: Text,
task: Text,
protocol: Text,
protocol_entries: Dict,
database: str,
task: str,
protocol: str,
protocol_entries: dict,
database_yml: Path,
version: dict | None = None,
) -> Union[type, None]:
registry=None,
) -> type | None:
"""Create new protocol class
Parameters
@@ -416,6 +406,8 @@ def create_protocol(
protocol_entries : dict
version : dict, optional
Annotation version as {"name": ..., "metadata": ...}
registry : Registry, optional
Registry used to resolve members of meta-protocols.
Returns
-------
@@ -423,7 +415,6 @@ def create_protocol(
"""
try:
base_class = getattr(
protocol_module, "Protocol" if task == "Protocol" else f"{task}Protocol"
@@ -466,7 +457,7 @@ def create_protocol(
msg = (
f"'{database}.{task}.{protocol}' found in {database_yml} does not define "
f"the 'scope' of speaker labels (file, database, or global). Setting it to 'file'."
)
)
print(msg)
metadata["scope"] = "file"
@@ -475,7 +466,6 @@ def create_protocol(
methods = dict()
for subset, subset_entries in protocol_entries.items():
if subset not in [
"files",
"train",
@@ -512,10 +502,10 @@ def create_protocol(
subset,
subset_entries,
database_yml,
registry=registry,
)
)
else:
methods[method_name] = functools.partialmethod(
subset_iter,
database=database,
+4 -9
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2016 CNRS
@@ -27,10 +24,10 @@
# Hervé BREDIN - http://herve.niderb.fr
from typing import Optional
from .protocol.protocol import Preprocessors
import warnings
from .protocol.protocol import Preprocessors
class Database:
"""Base database
@@ -49,7 +46,7 @@ class Database:
)
warnings.warn(msg)
super(Database, self).__init__()
super().__init__()
def register_protocol(self, task_name, protocol_name, protocol):
if not hasattr(self, "protocols_"):
@@ -74,9 +71,7 @@ class Database:
def get_protocols(self, task):
return sorted(self.protocols_[task].keys())
def get_protocol(
self, task, protocol, preprocessors: Optional[Preprocessors] = None
):
def get_protocol(self, task, protocol, preprocessors: Preprocessors | None = None):
return self.protocols_[task][protocol](preprocessors=preprocessors)
def __str__(self):
+11 -12
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2016- CNRS
@@ -31,7 +28,6 @@ import glob
import os
import warnings
from pathlib import Path
from typing import Text
from pyannote.database.protocol.protocol import ProtocolFile
@@ -57,12 +53,16 @@ def _scan_candidate(path: Path) -> list[Path]:
return [path] if path.is_file() else []
pattern = glob.escape(str(path)).replace("[*]", "*")
return [p for match in glob.iglob(pattern, recursive=True) if (p := Path(match)).is_file()]
return [
p
for match in glob.iglob(pattern, recursive=True)
if (p := Path(match)).is_file()
]
class FileFinder:
"""Database file finder.
"""Database file finder.
Retrieve media files by URI.
Parameters
@@ -71,16 +71,15 @@ class FileFinder:
Database registry. Defaults to `pyannote.database.registry`.
"""
def __init__(
self,
registry: Registry = None,
database_yml: Text = None):
def __init__(self, registry: Registry = None, database_yml: str = None):
super().__init__()
if registry is None:
if database_yml is None:
registry = global_registry
else:
warnings.warn("Passing `database.yml` to `FileFinder` is deprecated in favor of `registry`.")
warnings.warn(
"Passing `database.yml` to `FileFinder` is deprecated in favor of `registry`."
)
registry = Registry()
registry.load_database(database_yml)
self.registry = registry
+8 -10
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2020-2025 CNRS
@@ -33,10 +30,11 @@
import string
import warnings
from pathlib import Path
from typing import Any, Text
from typing import Any
import pandas as pd
from pyannote.core import Annotation, Timeline
from pyannote.database.protocol.protocol import ProtocolFile
from pyannote.database.util import load_lab, load_rttm, load_uem
@@ -66,7 +64,7 @@ def load_lst(file_lst):
List or uris
"""
with open(file_lst, mode="r") as fp:
with open(file_lst) as fp:
lines = fp.readlines()
return [line.strip() for line in lines]
@@ -105,7 +103,7 @@ class RTTMLoader:
(e.g. "/path/to/{database}/{subset}/{uri}.rttm")
"""
def __init__(self, path: Text = None):
def __init__(self, path: str = None):
super().__init__()
self.path = str(path)
@@ -156,7 +154,7 @@ class STMLoader:
self.placeholders_ = set(placeholders) - set([None])
if self.placeholders_:
self.loaded_: dict[str, "SegLST"] = dict()
self.loaded_: dict[str, SegLST] = dict()
return
if MEETEVAL_IS_AVAILABLE:
@@ -170,7 +168,7 @@ class STMLoader:
return
warnings.warn("MeetEval is not available, STM files cannot be loaded.")
self.loaded_: dict[str, "SegLST"] = dict()
self.loaded_: dict[str, SegLST] = dict()
def __call__(self, file: ProtocolFile) -> "SegLST":
uri = file["uri"]
@@ -223,7 +221,7 @@ class UEMLoader:
(e.g. "/path/to/{database}/{subset}/{uri}.uem")
"""
def __init__(self, path: Text = None):
def __init__(self, path: str = None):
super().__init__()
self.path = str(path)
@@ -273,7 +271,7 @@ class LABLoader:
...
"""
def __init__(self, path: Text = None):
def __init__(self, path: str = None):
super().__init__()
self.path = str(path)
+3 -8
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2016- CNRS
@@ -26,15 +23,14 @@
# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr
from .protocol import Protocol
from .collection import CollectionProtocol
from .protocol import Protocol
from .segmentation import SegmentationProtocol
from .speaker_diarization import SpeakerDiarizationProtocol
from .speaker_spotting import SpeakerSpottingProtocol
from .speaker_verification import SpeakerVerificationProtocol
from .speaker_identification import SpeakerIdentificationProtocol
from .speaker_recognition import SpeakerRecognitionProtocol
from .speaker_spotting import SpeakerSpottingProtocol
from .speaker_verification import SpeakerVerificationProtocol
__all__ = [
"Protocol",
@@ -46,4 +42,3 @@ __all__ = [
"SpeakerIdentificationProtocol",
"SpeakerRecognitionProtocol",
]
+4 -6
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2018-2020 CNRS
@@ -27,7 +24,8 @@
# Hervé BREDIN - http://herve.niderb.fr
from typing import Iterator, Dict
from collections.abc import Iterator
from .protocol import Protocol
@@ -90,9 +88,9 @@ class CollectionProtocol(Protocol):
"""
# this method should be overriden
def files_iter(self) -> Iterator[Dict]:
def files_iter(self) -> Iterator[dict]:
raise NotImplementedError()
# this allows Protocol.files() to iterate over the collection
def train_iter(self) -> Iterator[Dict]:
def train_iter(self) -> Iterator[dict]:
return self.files_iter()
+14 -32
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2016-2020 CNRS
@@ -33,23 +30,19 @@ Protocols
"""
import warnings
import collections
import threading
import itertools
from typing import Union, Dict, Iterator, Callable, Any, Text, Optional
try:
from typing import Literal
except ImportError:
from typing_extensions import Literal
import threading
import warnings
from collections.abc import Callable, Iterator
from typing import Any, Literal, Union
Subset = Literal["train", "development", "test"]
LEGACY_SUBSET_MAPPING = {"train": "trn", "development": "dev", "test": "tst"}
Scope = Literal["file", "database", "global"]
Preprocessor = Callable[["ProtocolFile"], Any]
Preprocessors = Dict[Text, Preprocessor]
Preprocessors = dict[str, Preprocessor]
class ProtocolFile(collections.abc.MutableMapping):
@@ -70,7 +63,7 @@ class ProtocolFile(collections.abc.MutableMapping):
"""
def __init__(self, precomputed: Union[Dict, "ProtocolFile"], lazy: Dict = None):
def __init__(self, precomputed: Union[dict, "ProtocolFile"], lazy: dict = None):
if lazy is None:
lazy = dict()
@@ -79,7 +72,7 @@ class ProtocolFile(collections.abc.MutableMapping):
# when 'precomputed' is a ProtocolFile, it may already contain lazy keys.
# we use 'precomputed' precomputed keys as precomputed keys
self._store: Dict = abs(precomputed)
self._store: dict = abs(precomputed)
# we handle the corner case where the intersection of 'precomputed' lazy keys
# and 'lazy' keys is not empty. this is currently achieved by "unlazying" the
@@ -90,7 +83,7 @@ class ProtocolFile(collections.abc.MutableMapping):
# we use the union of 'precomputed' lazy keys and provided 'lazy' keys as lazy keys
compound_lazy = dict(precomputed.lazy)
compound_lazy.update(lazy)
self.lazy: Dict = compound_lazy
self.lazy: dict = compound_lazy
else:
# when 'precomputed' is a Dict, we use it directly as precomputed keys
@@ -124,9 +117,7 @@ class ProtocolFile(collections.abc.MutableMapping):
def __getitem__(self, key):
with self.lock_:
if key in self.lazy and self.evaluating_[key] == 0:
# mark lazy key as being evaluated
self.evaluating_.update([key])
@@ -150,7 +141,6 @@ class ProtocolFile(collections.abc.MutableMapping):
def __setitem__(self, key, value):
with self.lock_:
if key in self.lazy:
del self.lazy[key]
@@ -158,7 +148,6 @@ class ProtocolFile(collections.abc.MutableMapping):
def __delitem__(self, key):
with self.lock_:
if key in self.lazy:
del self.lazy[key]
@@ -166,7 +155,6 @@ class ProtocolFile(collections.abc.MutableMapping):
def __iter__(self):
with self.lock_:
store_keys = list(self._store)
for key in store_keys:
yield key
@@ -220,7 +208,6 @@ class ProtocolFile(collections.abc.MutableMapping):
precomputed = {"uri": uris}
for key, value in abs(self).items():
if key == "uri":
continue
@@ -311,7 +298,7 @@ class Protocol:
(e.g. {'audio': '/path/to/{uri}.wav'})
"""
def __init__(self, preprocessors: Optional[Preprocessors] = None):
def __init__(self, preprocessors: Preprocessors | None = None):
super().__init__()
if preprocessors is None:
@@ -319,7 +306,6 @@ class Protocol:
self.preprocessors = dict()
for key, preprocessor in preprocessors.items():
if callable(preprocessor):
self.preprocessors[key] = preprocessor
@@ -337,21 +323,21 @@ class Protocol:
msg = f'"{key}" preprocessor is neither a callable nor a string.'
raise ValueError(msg)
def preprocess(self, current_file: Union[Dict, ProtocolFile]) -> ProtocolFile:
def preprocess(self, current_file: dict | ProtocolFile) -> ProtocolFile:
return ProtocolFile(current_file, lazy=self.preprocessors)
def __str__(self):
return self.__doc__
def train_iter(self) -> Iterator[Union[Dict, ProtocolFile]]:
def train_iter(self) -> Iterator[dict | ProtocolFile]:
"""Iterate over files in the training subset"""
raise NotImplementedError()
def development_iter(self) -> Iterator[Union[Dict, ProtocolFile]]:
def development_iter(self) -> Iterator[dict | ProtocolFile]:
"""Iterate over files in the development subset"""
raise NotImplementedError()
def test_iter(self) -> Iterator[Union[Dict, ProtocolFile]]:
def test_iter(self) -> Iterator[dict | ProtocolFile]:
"""Iterate over files in the test subset"""
raise NotImplementedError()
@@ -402,19 +388,16 @@ class Protocol:
"train_enrolment",
"train_trial",
]:
if not hasattr(self, method):
continue
def iterate():
try:
for file in getattr(self, method)():
yield file
yield from getattr(self, method)()
except (AttributeError, NotImplementedError):
return
for current_file in iterate():
# skip "files" that do not contain a "uri" entry.
# this happens for speaker verification trials that contain
# two nested files "file1" and "file2"
@@ -423,7 +406,6 @@ class Protocol:
continue
for current_file_ in current_file.files():
# corner case when the same file is yielded several times
uri = get_unique_identifier(current_file_)
if uri in yielded_uris:
+9 -18
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2016- CNRS
@@ -27,20 +24,15 @@
# Hervé BREDIN - http://herve.niderb.fr
from typing import Dict, Optional
from .protocol import Protocol
from .protocol import ProtocolFile
from .protocol import Subset
from .protocol import Preprocessor
from .protocol import Preprocessors
from pyannote.core import Annotation
from pyannote.core import Timeline
from pyannote.core import Segment
import functools
from pyannote.core import Annotation, Segment, Timeline
from .protocol import Preprocessor, Preprocessors, Protocol, ProtocolFile, Subset
def crop_annotated(
current_file: ProtocolFile, existing_preprocessor: Optional[Preprocessor] = None
current_file: ProtocolFile, existing_preprocessor: Preprocessor | None = None
) -> Timeline:
"""Preprocessor that crops 'annotated' according to 'duration'
@@ -75,14 +67,14 @@ def crop_annotated(
# crop 'annotated' to 'duration'
duration = Segment(0.0, duration)
if annotated and not annotated.extent() in duration:
if annotated and annotated.extent() not in duration:
return annotated.crop(duration, mode="intersection")
return annotated
def crop_annotation(
current_file: ProtocolFile, existing_preprocessor: Optional[Preprocessor] = None
current_file: ProtocolFile, existing_preprocessor: Preprocessor | None = None
) -> Annotation:
"""Preprocessor that crops 'annotation' by 'annotated'
@@ -215,7 +207,7 @@ class SegmentationProtocol(Protocol):
filename2
"""
def __init__(self, preprocessors: Optional[Preprocessors] = None):
def __init__(self, preprocessors: Preprocessors | None = None):
if preprocessors is None:
preprocessors = dict()
@@ -234,7 +226,7 @@ class SegmentationProtocol(Protocol):
super().__init__(preprocessors=preprocessors)
def stats(self, subset: Subset = "train") -> Dict:
def stats(self, subset: Subset = "train") -> dict:
"""Obtain global statistics on a given subset
Parameters
@@ -263,7 +255,6 @@ class SegmentationProtocol(Protocol):
labels = {}
for item in getattr(self, subset)():
annotated = get_annotated(item)
annotated_duration += annotated.duration()
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2016- CNRS
@@ -28,6 +25,7 @@
from .segmentation import SegmentationProtocol
class SpeakerDiarizationProtocol(SegmentationProtocol):
"""A protocol for speaker diarization experiments
@@ -122,4 +120,4 @@ class SpeakerDiarizationProtocol(SegmentationProtocol):
filename2
"""
pass
pass
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2017 CNRS
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2016-2020 CNRS
@@ -44,76 +41,76 @@ class SpeakerRecognitionProtocol(Protocol):
def trn_iter(self):
raise NotImplementedError(
"Custom speaker recognition protocol " 'should implement "trn_iter".'
'Custom speaker recognition protocol should implement "trn_iter".'
)
def trn_enroll_iter(self):
raise NotImplementedError(
"Custom speaker recognition protocol " 'should implement "trn_enroll_iter".'
'Custom speaker recognition protocol should implement "trn_enroll_iter".'
)
def trn_test_iter(self):
raise NotImplementedError(
"Custom speaker recognition protocol " 'should implement "trn_test_iter".'
'Custom speaker recognition protocol should implement "trn_test_iter".'
)
def trn_keys(self):
raise NotImplementedError(
"Custom speaker recognition protocol " 'should implement "trn_keys".'
'Custom speaker recognition protocol should implement "trn_keys".'
)
def dev_enroll_iter(self):
raise NotImplementedError(
"Custom speaker recognition protocol " 'should implement "dev_enroll_iter".'
'Custom speaker recognition protocol should implement "dev_enroll_iter".'
)
def dev_test_iter(self):
raise NotImplementedError(
"Custom speaker recognition protocol " 'should implement "dev_test_iter".'
'Custom speaker recognition protocol should implement "dev_test_iter".'
)
def dev_keys(self):
raise NotImplementedError(
"Custom speaker recognition protocol " 'should implement "dev_keys".'
'Custom speaker recognition protocol should implement "dev_keys".'
)
def tst_enroll_iter(self):
raise NotImplementedError(
"Custom speaker recognition protocol " 'should implement "tst_enroll_iter".'
'Custom speaker recognition protocol should implement "tst_enroll_iter".'
)
def tst_test_iter(self):
raise NotImplementedError(
"Custom speaker recognition protocol " 'should implement "tst_test_iter".'
'Custom speaker recognition protocol should implement "tst_test_iter".'
)
def tst_keys(self):
raise NotImplementedError(
"Custom speaker recognition protocol " 'should implement tst_keys".'
'Custom speaker recognition protocol should implement tst_keys".'
)
def train(self, yield_name=False):
"""Iterate over the training set
This will yield dictionaries with the followings keys:
This will yield dictionaries with the followings keys:
* database: str
unique database identifier
* uri: str
unique recording identifier
* channel: int
index of resource channel to use
* speaker: str
unique speaker identifier
* database: str
unique database identifier
* uri: str
unique recording identifier
* channel: int
index of resource channel to use
* speaker: str
unique speaker identifier
as well as keys coming from the provided preprocessors.
as well as keys coming from the provided preprocessors.
Usage
-----
>>> for item in protocol.train():
... uri = item['uri']
... channel = item['channel']
... speaker = item['speaker']
Usage
-----
>>> for item in protocol.train():
... uri = item['uri']
... channel = item['channel']
... speaker = item['speaker']
"""
generator = self.trn_iter()
@@ -127,22 +124,22 @@ Usage
def train_enroll(self, yield_name=True):
"""Iterate over the training set enrollments
This will yield dictionaries with the followings keys:
This will yield dictionaries with the followings keys:
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
as well as keys coming from the provided preprocessors.
as well as keys coming from the provided preprocessors.
Usage
-----
>>> for item in protocol.train_enroll():
... uri = item['uri']
... channel = item['channel']
Usage
-----
>>> for item in protocol.train_enroll():
... uri = item['uri']
... channel = item['channel']
"""
generator = self.trn_enroll_iter()
@@ -156,22 +153,22 @@ Usage
def train_test(self, yield_name=True):
"""Iterate over the training set tests
This will yield dictionaries with the followings keys:
This will yield dictionaries with the followings keys:
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
as well as keys coming from the provided preprocessors.
as well as keys coming from the provided preprocessors.
Usage
-----
>>> for item in protocol.train_test():
... uri = item['uri']
... channel = item['channel']
Usage
-----
>>> for item in protocol.train_test():
... uri = item['uri']
... channel = item['channel']
"""
generator = self.trn_test_iter()
@@ -188,22 +185,22 @@ Usage
def development_enroll(self, yield_name=True):
"""Iterate over the development set enrollments
This will yield dictionaries with the followings keys:
This will yield dictionaries with the followings keys:
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
as well as keys coming from the provided preprocessors.
as well as keys coming from the provided preprocessors.
Usage
-----
>>> for item in protocol.development_enroll():
... uri = item['uri']
... channel = item['channel']
Usage
-----
>>> for item in protocol.development_enroll():
... uri = item['uri']
... channel = item['channel']
"""
generator = self.dev_enroll_iter()
@@ -217,22 +214,22 @@ Usage
def development_test(self, yield_name=True):
"""Iterate over the development set tests
This will yield dictionaries with the followings keys:
This will yield dictionaries with the followings keys:
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
as well as keys coming from the provided preprocessors.
as well as keys coming from the provided preprocessors.
Usage
-----
>>> for item in protocol.development_test():
... uri = item['uri']
... channel = item['channel']
Usage
-----
>>> for item in protocol.development_test():
... uri = item['uri']
... channel = item['channel']
"""
generator = self.dev_test_iter()
@@ -249,25 +246,25 @@ Usage
def test_enroll(self, yield_name=True):
"""Iterate over the test set targets
This will yield dictionaries with the followings keys:
This will yield dictionaries with the followings keys:
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
* speaker: str
unique speaker identifier
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
* speaker: str
unique speaker identifier
as well as keys coming from the provided preprocessors.
as well as keys coming from the provided preprocessors.
Usage
-----
>>> for item in protocol.test_enroll():
... uri = item['uri']
... channel = item['channel']
... speaker = item['speaker']
Usage
-----
>>> for item in protocol.test_enroll():
... uri = item['uri']
... channel = item['channel']
... speaker = item['speaker']
"""
generator = self.tst_enroll_iter()
@@ -281,22 +278,22 @@ Usage
def test_test(self, yield_name=True):
"""Iterate over the test set tests
This will yield dictionaries with the followings keys:
This will yield dictionaries with the followings keys:
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
* database: str
unique database identifier
* uri: str
uniform (or unique) resource identifier
* channel: int
index of resource channel to use
as well as keys coming from the provided preprocessors.
as well as keys coming from the provided preprocessors.
Usage
-----
>>> for item in protocol.test_test():
... uri = item['uri']
... channel = item['channel']
Usage
-----
>>> for item in protocol.test_test():
... uri = item['uri']
... channel = item['channel']
"""
generator = self.tst_test_iter()
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2017-2020 CNRS
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2017-2020 CNRS
@@ -27,10 +24,10 @@
# Hervé BREDIN - http://herve.niderb.fr
from typing import Dict, Iterator
from collections.abc import Iterator
from .protocol import LEGACY_SUBSET_MAPPING, Subset
from .speaker_diarization import SpeakerDiarizationProtocol
from .protocol import Subset
from .protocol import LEGACY_SUBSET_MAPPING
class SpeakerVerificationProtocol(SpeakerDiarizationProtocol):
@@ -134,7 +131,7 @@ class SpeakerVerificationProtocol(SpeakerDiarizationProtocol):
As such, they also define regular `{subset}` methods.
"""
def subset_trial_helper(self, subset: Subset) -> Iterator[Dict]:
def subset_trial_helper(self, subset: Subset) -> Iterator[dict]:
try:
trials = getattr(self, f"{subset}_trial_iter")()
@@ -156,23 +153,23 @@ class SpeakerVerificationProtocol(SpeakerDiarizationProtocol):
trial["file2"] = self.preprocess(trial["file2"])
yield trial
def train_trial_iter(self) -> Iterator[Dict]:
def train_trial_iter(self) -> Iterator[dict]:
"""Iterate over trials in the train subset"""
raise NotImplementedError()
def development_trial_iter(self) -> Iterator[Dict]:
def development_trial_iter(self) -> Iterator[dict]:
"""Iterate over trials in the development subset"""
raise NotImplementedError()
def test_trial_iter(self) -> Iterator[Dict]:
def test_trial_iter(self) -> Iterator[dict]:
"""Iterate over trials in the test subset"""
raise NotImplementedError()
def train_trial(self) -> Iterator[Dict]:
def train_trial(self) -> Iterator[dict]:
return self.subset_trial_helper("train")
def development_trial(self) -> Iterator[Dict]:
def development_trial(self) -> Iterator[dict]:
return self.subset_trial_helper("development")
def test_trial(self) -> Iterator[Dict]:
def test_trial(self) -> Iterator[dict]:
return self.subset_trial_helper("test")
+40 -31
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2022- CNRS
@@ -28,16 +25,17 @@
# Hervé BREDIN - http://herve.niderb.fr
# Alexis PLAQUET
from enum import Enum
import os
from pathlib import Path
from typing import Dict, List, Set, Optional, Text, Tuple, Type, Union
import warnings
from enum import Enum
from pathlib import Path
import yaml
from pyannote.database.protocol.protocol import Preprocessors, Protocol
from .custom import create_protocol, get_init
from .database import Database
import yaml
# controls what to do in case of protocol name conflict
@@ -47,7 +45,7 @@ class LoadingMode(Enum):
ERROR = 2 # raise an error
def _apply_version(entries: Dict, version: Text) -> Dict:
def _apply_version(entries: dict, version: str) -> dict:
"""Replace {version} placeholder in protocol entries
Parameters
@@ -68,16 +66,18 @@ def _apply_version(entries: Dict, version: Text) -> Dict:
return value.replace("{version}", version) if isinstance(value, str) else value
return {
key: {sub(k): sub(v) for k, v in value.items()} if isinstance(value, dict) else value
key: {sub(k): sub(v) for k, v in value.items()}
if isinstance(value, dict)
else value
for key, value in entries.items()
}
def _expand_versions(
db_name: Text,
task_name: Text,
protocol: Text,
protocol_entries: Dict,
db_name: str,
task_name: str,
protocol: str,
protocol_entries: dict,
database_yml: Path,
) -> list[tuple[str, dict, dict | None]]:
"""Expand protocol into one protocol per version
@@ -125,7 +125,9 @@ def _expand_versions(
)
raise ValueError(msg)
invalid = [str(n) for n, m in versions.items() if not (m is None or isinstance(m, dict))]
invalid = [
str(n) for n, m in versions.items() if not (m is None or isinstance(m, dict))
]
if invalid:
msg = f"{where}: metadata of versions {invalid} must be a mapping (e.g. {{closing: 0.2}})."
raise ValueError(msg)
@@ -143,7 +145,8 @@ def _expand_versions(
raise ValueError(msg)
return [
(f"{protocol}@{v['name']}", _apply_version(entries, v["name"]), v) for v in versions
(f"{protocol}@{v['name']}", _apply_version(entries, v["name"]), v)
for v in versions
]
@@ -243,7 +246,7 @@ class Registry:
# }
# }
# }
self.configs: Dict[Path, Dict] = dict()
self.configs: dict[Path, dict] = dict()
# Content of the "Database" root item (= where to find file content),
# with paths as written in database.yml. Only used by FileFinder.
@@ -274,11 +277,11 @@ class Registry:
# "DatabaseB": pyannote.database.registry.DatabaseB,
# "DatabaseC": pyannote.database.registry.DatabaseC,
# "X": pyannote.database.registry.X}
self.databases: Dict[Text, Type] = dict()
self.databases: dict[str, type] = dict()
def load_database(
self,
path: Union[Text, Path],
path: str | Path,
mode: LoadingMode = LoadingMode.OVERRIDE,
):
"""Load YAML configuration file into the registry
@@ -302,9 +305,9 @@ class Registry:
def _load_database_helper(
self,
database_yml: Union[Text, Path],
database_yml: str | Path,
mode: LoadingMode = LoadingMode.KEEP,
loading: Set[Path] = set(),
loading: set[Path] = set(),
):
"""Helper function for recursive loading
@@ -329,7 +332,7 @@ class Registry:
loading.add(database_yml)
# load configuration
with open(database_yml, "r") as f:
with open(database_yml) as f:
config = yaml.load(f, Loader=yaml.SafeLoader)
# load every requirement
@@ -404,7 +407,7 @@ class Registry:
return database(**kwargs)
def get_protocol(
self, name, preprocessors: Optional[Preprocessors] = None
self, name, preprocessors: Preprocessors | None = None
) -> Protocol:
"""Get protocol by full name
@@ -471,7 +474,7 @@ class Registry:
self,
db_name,
db_entries: dict,
database_yml: Union[Text, Path] = None,
database_yml: str | Path = None,
mode: LoadingMode = LoadingMode.OVERRIDE,
):
"""Load all protocols from this database into the registry.
@@ -489,7 +492,7 @@ class Registry:
db_name = str(db_name)
# maps tuple (task,protocol) to the custom protocol class
protocols: Dict[Tuple[Text, Text], Type] = dict()
protocols: dict[tuple[str, str], type] = dict()
for task_name, task_entries in db_entries.items():
for protocol, protocol_entries in task_entries.items():
@@ -515,6 +518,7 @@ class Registry:
entries,
database_yml,
version=version,
registry=self,
)
if CustomProtocol is None:
continue
@@ -523,8 +527,13 @@ class Registry:
# expanded[0] is the default (first listed) version, e.g. "Raw@original":
default_protocol, _, default_version = expanded[0]
if default_version is not None and (task_name, default_protocol) in protocols:
protocols[(task_name, protocol)] = protocols[(task_name, default_protocol)]
if (
default_version is not None
and (task_name, default_protocol) in protocols
):
protocols[(task_name, protocol)] = protocols[
(task_name, default_protocol)
]
# If needed, merge old protocols dict with the new one (according to current override rules)
if db_name in self.databases:
@@ -558,7 +567,7 @@ class Registry:
)
def _env_config_paths() -> List[Path]:
def _env_config_paths() -> list[Path]:
"""Parse PYANNOTE_DATABASE_CONFIG environment variable
PYANNOTE_DATABASE_CONFIG may contain multiple paths separation by ";".
@@ -579,7 +588,7 @@ def _env_config_paths() -> List[Path]:
return paths
def _find_default_ymls() -> List[Path]:
def _find_default_ymls() -> list[Path]:
"""Get paths to default YAML configuration files
* $HOME/.pyannote/database.yml
@@ -592,7 +601,7 @@ def _find_default_ymls() -> List[Path]:
List of existing default YAML configuration files
"""
paths: List[Path] = []
paths: list[Path] = []
home_db_yml = Path("~/.pyannote/database.yml").expanduser()
if home_db_yml.is_file():
@@ -608,8 +617,8 @@ def _find_default_ymls() -> List[Path]:
def _merge_protocols_inplace(
new_protocols: Dict[Tuple[Text, Text], Type],
old_protocols: Dict[Tuple[Text, Text], Type],
new_protocols: dict[tuple[str, str], type],
old_protocols: dict[tuple[str, str], type],
mode: LoadingMode,
db_name: str,
database_yml: str,
+7 -17
View File
@@ -1,6 +1,3 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2016-2020 CNRS
@@ -26,20 +23,13 @@
# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr
import yaml
from pathlib import Path
import warnings
import pandas as pd
from pyannote.core import Segment, Timeline, Annotation
from .protocol.protocol import ProtocolFile
from pyannote.core import Annotation, Segment, Timeline
from typing import Text
from typing import Union
from typing import Dict
from typing import List
DatabaseName = Text
PathTemplate = Text
DatabaseName = str
PathTemplate = str
def get_unique_identifier(item):
@@ -332,7 +322,7 @@ def load_lst(file_lst):
List or uris
"""
with open(file_lst, mode="r") as fp:
with open(file_lst) as fp:
lines = fp.readlines()
return [line.strip() for line in lines]
@@ -351,7 +341,7 @@ def load_mapping(mapping_txt):
{1st field: 2nd field} dictionary
"""
with open(mapping_txt, mode="r") as fp:
with open(mapping_txt) as fp:
lines = fp.readlines()
mapping = dict()
@@ -362,7 +352,7 @@ def load_mapping(mapping_txt):
return mapping
class LabelMapper(object):
class LabelMapper:
"""Label mapper for use as pyannote.database preprocessor
Parameters
+1 -1
View File
@@ -10,7 +10,7 @@ Protocols:
train:
uri: lists/train.lst
speaker: rttms/train.rttm
transcription: _ctms/{uri}.ctm
transcription: stms/{uri}.stm
domain: mapping/domain.map
SpeakerDiarization:
+2
View File
@@ -0,0 +1,2 @@
filename1 A speaker 0.2 0.5 hello
filename1 A speaker 0.3 0.7 world
+3
View File
@@ -0,0 +1,3 @@
filename2 A speaker 0.2 0.5 how
filename2 A speaker 0.5 0.9 are
filename2 A speaker 0.9 1.1 you
-86
View File
@@ -1,86 +0,0 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2023- CNRS
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr
from pyannote.database import registry
from pyannote.database.protocol import CollectionProtocol
from pyannote.database.protocol import Protocol
from pyannote.database.protocol import SpeakerDiarizationProtocol
from pyannote.database.protocol import SpeakerVerificationProtocol
assert "MyDatabase" in registry.databases
database = registry.get_database("MyDatabase")
tasks = database.get_tasks()
assert "Collection" in tasks
assert "Protocol" in tasks
assert "SpeakerDiarization" in tasks
assert "SpeakerVerification" in tasks
assert "MyCollection" in database.get_protocols("Collection")
assert "MyProtocol" in database.get_protocols("Protocol")
assert "MySpeakerDiarization" in database.get_protocols("SpeakerDiarization")
assert "MySpeakerVerification" in database.get_protocols("SpeakerVerification")
collection = registry.get_protocol("MyDatabase.Collection.MyCollection")
assert isinstance(collection, CollectionProtocol)
protocol = registry.get_protocol("MyDatabase.Protocol.MyProtocol")
assert isinstance(protocol, Protocol)
speaker_diarization = registry.get_protocol(
"MyDatabase.SpeakerDiarization.MySpeakerDiarization"
)
assert isinstance(speaker_diarization, SpeakerDiarizationProtocol)
speaker_verification = registry.get_protocol(
"MyDatabase.SpeakerVerification.MySpeakerVerification"
)
assert isinstance(speaker_verification, SpeakerVerificationProtocol)
files = list(collection.files())
assert len(files) == 2
files = list(protocol.files())
assert len(files) == 2
files = list(speaker_diarization.files())
assert len(files) == 2
files = list(speaker_verification.files())
assert len(files) == 2
meta_protocol = registry.get_protocol("X.SpeakerDiarization.MyMetaProtocol")
files = list(meta_protocol.train())
assert len(files) == 2
files = list(meta_protocol.development())
assert len(files) == 4
+59
View File
@@ -0,0 +1,59 @@
# Copyright (c) 2025- pyannoteAI
import subprocess
import sys
from importlib.metadata import entry_points
from pathlib import Path
import pytest
def test_cli_help():
pytest.importorskip("typer")
result = subprocess.run(
[str(Path(sys.executable).with_name("pyannote-database")), "--help"],
capture_output=True,
text=True,
check=True,
)
assert "Usage:" in result.stdout
def test_loader_entry_points():
loaders = {
ep.name: ep.load() for ep in entry_points(group="pyannote.database.loader")
}
assert {".rttm", ".uem", ".map", ".lab", ".stm"} <= loaders.keys()
assert all(callable(loader) for loader in loaders.values())
@pytest.mark.parametrize("templated", [False, True])
def test_stm_loader(tmp_path, templated):
pytest.importorskip("meeteval")
from meeteval.io.seglst import SegLST
from pyannote.database.loader import STMLoader
path = tmp_path / "sample.stm"
path.write_text("sample 1 speaker 0.0 1.0 hello world\n")
loader = STMLoader(tmp_path / "{uri}.stm" if templated else path)
transcript = loader({"uri": "sample"})
assert isinstance(transcript, SegLST)
assert [segment["words"] for segment in transcript] == ["hello world"]
assert list(loader({"uri": "sample"})) == list(transcript)
def test_missing_transcription_extra():
code = """
import sys
import warnings
sys.modules['meeteval'] = None
from pyannote.database.loader import STMLoader
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter('always')
loader = STMLoader('sample.stm')
result = loader({'uri': 'sample'})
assert result is None
assert any('MeetEval' in str(w.message) for w in caught)
"""
subprocess.run([sys.executable, "-c", code], check=True)
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python
# The MIT License (MIT)
# Copyright (c) 2023- CNRS
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr
from pyannote.database.protocol import (
CollectionProtocol,
Protocol,
SpeakerDiarizationProtocol,
SpeakerVerificationProtocol,
)
def test_configured_protocols():
from pathlib import Path
from pyannote.database.registry import Registry
registry = Registry()
registry.load_database(Path(__file__).parent / "data/database.yml")
assert "MyDatabase" in registry.databases
database = registry.get_database("MyDatabase")
tasks = database.get_tasks()
assert "Collection" in tasks
assert "Protocol" in tasks
assert "SpeakerDiarization" in tasks
assert "SpeakerVerification" in tasks
assert "MyCollection" in database.get_protocols("Collection")
assert "MyProtocol" in database.get_protocols("Protocol")
assert "MySpeakerDiarization" in database.get_protocols("SpeakerDiarization")
assert "MySpeakerVerification" in database.get_protocols("SpeakerVerification")
collection = registry.get_protocol("MyDatabase.Collection.MyCollection")
assert isinstance(collection, CollectionProtocol)
protocol = registry.get_protocol("MyDatabase.Protocol.MyProtocol")
assert isinstance(protocol, Protocol)
speaker_diarization = registry.get_protocol(
"MyDatabase.SpeakerDiarization.MySpeakerDiarization"
)
assert isinstance(speaker_diarization, SpeakerDiarizationProtocol)
speaker_verification = registry.get_protocol(
"MyDatabase.SpeakerVerification.MySpeakerVerification"
)
assert isinstance(speaker_verification, SpeakerVerificationProtocol)
files = list(collection.files())
assert len(files) == 2
files = list(protocol.files())
assert len(files) == 2
files = list(speaker_diarization.files())
assert len(files) == 2
files = list(speaker_verification.files())
assert len(files) == 2
meta_protocol = registry.get_protocol("X.SpeakerDiarization.MyMetaProtocol")
files = list(meta_protocol.train())
assert len(files) == 2
files = list(meta_protocol.development())
assert len(files) == 4
+16 -9
View File
@@ -1,5 +1,4 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
@@ -29,10 +28,12 @@
# Hervé BREDIN - http://herve.niderb.fr
import warnings
import pytest
from pyannote.database.registry import LoadingMode, _merge_protocols_inplace
def test_override_merging_disjoint():
protocols1 = {
("Task1", "Protocol1"): None,
@@ -45,33 +46,39 @@ def test_override_merging_disjoint():
warnings.simplefilter("error") # expect no warning
_merge_protocols_inplace(protocols1, protocols2, LoadingMode.KEEP, "", "")
assert ("Task1", "Protocol1",) in protocols1
assert ("OtherTask", "Protocol1",) in protocols1
assert (
"Task1",
"Protocol1",
) in protocols1
assert (
"OtherTask",
"Protocol1",
) in protocols1
assert len(protocols1) == 2
def test_override_merging_identical():
protocols2 = {
("Task1", "Protocol1"): None,
} # the "old" protocols dict. KEEP override options will keep these entries.
} # the "old" protocols dict. KEEP override options will keep these entries.
# Expect warning and protocols1 to become protocols2 (keep old value)
protocols1 = {
("Task1", "Protocol1"): 42,
}
with pytest.warns(Warning) as w:
with pytest.warns(Warning):
_merge_protocols_inplace(protocols1, protocols2, LoadingMode.KEEP, "", "")
assert ("Task1", "Protocol1") in protocols1
assert protocols1[("Task1", "Protocol1")] == None
assert protocols1[("Task1", "Protocol1")] is None
assert len(protocols1) == 1
# Expect warning and protocols1 to keep its value (use new value)
protocols1 = {
("Task1", "Protocol1"): 42,
}
with pytest.warns(Warning) as w:
with pytest.warns(Warning):
_merge_protocols_inplace(protocols1, protocols2, LoadingMode.OVERRIDE, "", "")
assert ("Task1", "Protocol1") in protocols1
assert protocols1[("Task1", "Protocol1")] == 42
assert len(protocols1) == 1
+22 -10
View File
@@ -1,5 +1,4 @@
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
@@ -29,13 +28,26 @@
# Alexis PLAQUET
from pyannote.database import registry
from pathlib import Path
protocol = registry.get_protocol('MyDatabase.SpeakerVerification.MySpeakerVerification')
for elt in protocol.train_trial():
print(elt)
print(elt['file1']['try_with'])
file1_annotation = elt['file1']['annotation']
print('annotation : ', file1_annotation)
file1_annotated = elt['file1']['annotated']
print('annotated : ', file1_annotated)
from pyannote.core import Annotation, Timeline
from pyannote.database.registry import Registry
def test_verification_trials():
registry = Registry()
registry.load_database(Path(__file__).parent / "data/database.yml")
protocol = registry.get_protocol(
"MyDatabase.SpeakerVerification.MySpeakerVerification"
)
trials = list(protocol.train_trial())
assert len(trials) == 1
trial = trials[0]
assert not trial["reference"]
for key, uri in (("file1", "filename1"), ("file2", "filename2")):
file = trial[key]
assert file["uri"] == uri
assert isinstance(file["annotation"], Annotation)
assert isinstance(file["annotated"], Timeline)
assert file["try_with"].duration() > 0
+37 -10
View File
@@ -23,7 +23,9 @@ def _duration(file):
def _protocols(database):
return set(global_registry.get_database(database).get_protocols("SpeakerDiarization"))
return set(
global_registry.get_database(database).get_protocols("SpeakerDiarization")
)
@pytest.fixture(scope="module")
@@ -96,8 +98,12 @@ Protocols:
def test_selecting_a_version_changes_annotation_only(versioned_dataset):
"""Selecting a version changes the annotation content, not which
recordings are covered."""
original = global_registry.get_protocol("VersionedTestDB.SpeakerDiarization.Raw@original")
refined = global_registry.get_protocol("VersionedTestDB.SpeakerDiarization.Raw@refined")
original = global_registry.get_protocol(
"VersionedTestDB.SpeakerDiarization.Raw@original"
)
refined = global_registry.get_protocol(
"VersionedTestDB.SpeakerDiarization.Raw@refined"
)
assert sorted(f["uri"] for f in original.train()) == ["u1", "u2"]
assert sorted(f["uri"] for f in refined.train()) == ["u1", "u2"]
@@ -121,7 +127,11 @@ def test_one_protocol_per_version_and_default_is_first_listed(versioned_dataset)
("Raw", {"name": "original", "metadata": {}}),
("Other", {"name": "refined", "metadata": {"closing": 0.2}}),
):
files = list(global_registry.get_protocol(f"VersionedTestDB.SpeakerDiarization.{name}").train())
files = list(
global_registry.get_protocol(
f"VersionedTestDB.SpeakerDiarization.{name}"
).train()
)
assert sorted(f["uri"] for f in files) == ["u1", "u2"]
assert all(f["version"] == default for f in files)
@@ -129,9 +139,15 @@ def test_one_protocol_per_version_and_default_is_first_listed(versioned_dataset)
def test_versions_without_placeholder_are_still_expanded(versioned_dataset):
"""Declaring `versions:` is enough to version a protocol: paths without
`{version}` are simply shared by every version."""
assert {"Shared", "Shared@original", "Shared@refined"} <= _protocols("VersionedTestDB")
assert {"Shared", "Shared@original", "Shared@refined"} <= _protocols(
"VersionedTestDB"
)
files = list(global_registry.get_protocol("VersionedTestDB.SpeakerDiarization.Shared@refined").train())
files = list(
global_registry.get_protocol(
"VersionedTestDB.SpeakerDiarization.Shared@refined"
).train()
)
assert all(f["version"]["name"] == "refined" and _duration(f) == 1.0 for f in files)
@@ -150,7 +166,9 @@ def test_unversioned_protocol_stays_flat(versioned_dataset):
def test_compound_member_with_version(versioned_dataset):
"""X meta-protocol members can pin an annotation version with @."""
compound = global_registry.get_protocol("X.SpeakerDiarization.VersionedTestCompound")
compound = global_registry.get_protocol(
"X.SpeakerDiarization.VersionedTestCompound"
)
files = list(compound.train())
assert sorted(f["uri"] for f in files) == ["u1", "u2"]
@@ -175,14 +193,23 @@ def test_invalid_version_selection_is_rejected(versioned_dataset, name, match):
"protocol_block, match",
[
# "version" is a reserved subset entry, even without `versions:`
("Raw:\n train:\n version: not_allowed", r"reserved for annotation versioning"),
(
"Raw:\n train:\n version: not_allowed",
r"reserved for annotation versioning",
),
# "@" is reserved to select a version
("Foo@Bar:\n scope: file", r"Foo@Bar.*must not contain '@'"),
# versions map names to (optional) metadata mappings
("Raw:\n versions: [original, refined]", r"must be a non-empty mapping"),
("Raw:\n versions: {}", r"must be a non-empty mapping"),
("Raw:\n versions: {original, refined: 0.2}", r"metadata of versions \['refined'\]"),
("Raw:\n versions: {original, refined: 0}", r"metadata of versions \['refined'\]"),
(
"Raw:\n versions: {original, refined: 0.2}",
r"metadata of versions \['refined'\]",
),
(
"Raw:\n versions: {original, refined: 0}",
r"metadata of versions \['refined'\]",
),
# "." would make the protocol class unpicklable
("Raw:\n versions: {original, v2.1}", r"must not contain '\.'.*v2\.1"),
],
Generated
-1397
View File
File diff suppressed because it is too large. Load diff