mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
feat: add support for PYANNOTE_SKIP_DEPENDENCY_CHECK environment variable
This commit is contained in:
1 parent
bec0b4f5cd
commit
96c345fe98
5 files changed
+50
-60
No files matched your search
@@ -535,12 +535,6 @@ def benchmark(
|
||||
help="Show progress",
|
||||
),
|
||||
] = False,
|
||||
skip_dependency_check: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="Skip dependency check when loading pipeline. Use at your own risk."
|
||||
),
|
||||
] = False,
|
||||
per_file: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
@@ -558,9 +552,7 @@ def benchmark(
|
||||
"""
|
||||
|
||||
# load pretrained pipeline
|
||||
pretrained_pipeline = Pipeline.from_pretrained(
|
||||
pipeline, cache_dir=cache, skip_dependencies=skip_dependency_check
|
||||
)
|
||||
pretrained_pipeline = Pipeline.from_pretrained(pipeline, cache_dir=cache)
|
||||
if pretrained_pipeline is None:
|
||||
print(f"Could not load pretrained pipeline from {pipeline}.")
|
||||
raise typer.exit(code=1)
|
||||
|
||||
@@ -504,7 +504,6 @@ class Model(lightning.LightningModule):
|
||||
revision: str | None = None,
|
||||
token: str | bool | None = None,
|
||||
cache_dir: Path | str | None = None,
|
||||
skip_dependencies: bool = False,
|
||||
**kwargs,
|
||||
) -> Optional["Model"]:
|
||||
"""Load pretrained model
|
||||
@@ -534,9 +533,6 @@ class Model(lightning.LightningModule):
|
||||
kwargs: optional
|
||||
Any extra keyword args needed to init the model.
|
||||
Can also be used to override saved hyperparameter values.
|
||||
skip_dependencies : bool, optional
|
||||
If True, skip dependency check. Defaults to False.
|
||||
Use at your own risk, as this may lead to unexpected behavior.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -605,10 +601,9 @@ class Model(lightning.LightningModule):
|
||||
# load checkpoint using lightning
|
||||
loaded_checkpoint = pl_load(path_to_model_checkpoint, map_location=map_location)
|
||||
|
||||
if not skip_dependencies:
|
||||
# check that the checkpoint is compatible with the current version
|
||||
versions = loaded_checkpoint["pyannote.audio"]["versions"]
|
||||
check_dependencies({"pyannote.audio": versions["pyannote.audio"]}, "Model")
|
||||
# check that the checkpoint is compatible with the current version
|
||||
versions = loaded_checkpoint["pyannote.audio"]["versions"]
|
||||
check_dependencies({"pyannote.audio": versions["pyannote.audio"]}, "Model")
|
||||
|
||||
# obtain model class from the checkpoint
|
||||
module_name: str = loaded_checkpoint["pyannote.audio"]["architecture"]["module"]
|
||||
|
||||
@@ -51,7 +51,6 @@ def expand_subfolders(
|
||||
parent_revision: str | None = None,
|
||||
cache_dir: Path | str | None = None,
|
||||
token: str | None = None,
|
||||
skip_dependencies: bool = False,
|
||||
) -> None:
|
||||
"""Expand $model subfolders in config
|
||||
|
||||
@@ -71,9 +70,6 @@ def expand_subfolders(
|
||||
Huggingface token to be used for downloading from Huggingface hub.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where files downloaded from Huggingface hub are stored.
|
||||
skip_dependencies : bool, optional
|
||||
If True, skip dependency check. Defaults to False.
|
||||
Use at your own risk, as this may lead to unexpected behavior.
|
||||
"""
|
||||
|
||||
if isinstance(config, dict):
|
||||
@@ -94,7 +90,6 @@ def expand_subfolders(
|
||||
"subfolder": subfolder,
|
||||
"token": token,
|
||||
"cache_dir": cache_dir,
|
||||
"skip_dependencies": skip_dependencies,
|
||||
}
|
||||
else:
|
||||
expand_subfolders(
|
||||
@@ -103,7 +98,6 @@ def expand_subfolders(
|
||||
parent_revision=parent_revision,
|
||||
token=token,
|
||||
cache_dir=cache_dir,
|
||||
skip_dependencies=skip_dependencies,
|
||||
)
|
||||
|
||||
elif isinstance(config, list):
|
||||
@@ -124,7 +118,6 @@ def expand_subfolders(
|
||||
"subfolder": subfolder,
|
||||
"token": token,
|
||||
"cache_dir": cache_dir,
|
||||
"skip_dependencies": skip_dependencies,
|
||||
}
|
||||
|
||||
else:
|
||||
@@ -134,7 +127,6 @@ def expand_subfolders(
|
||||
parent_revision=parent_revision,
|
||||
token=token,
|
||||
cache_dir=cache_dir,
|
||||
skip_dependencies=skip_dependencies,
|
||||
)
|
||||
|
||||
|
||||
@@ -147,7 +139,6 @@ class Pipeline(_Pipeline):
|
||||
hparams_file: str | Path | None = None,
|
||||
token: str | bool | None = None,
|
||||
cache_dir: Path | str | None = None,
|
||||
skip_dependencies: bool = False,
|
||||
) -> Optional["Pipeline"]:
|
||||
"""Load pretrained pipeline
|
||||
|
||||
@@ -166,9 +157,6 @@ class Pipeline(_Pipeline):
|
||||
Token to be used for the download.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where cached files are stored.
|
||||
skip_dependencies : bool, optional
|
||||
If True, skip dependency check. Defaults to False.
|
||||
Use at your own risk, as this may lead to unexpected behavior.
|
||||
"""
|
||||
|
||||
# if checkpoint is a dict, assume it is the actual content of
|
||||
@@ -233,7 +221,6 @@ class Pipeline(_Pipeline):
|
||||
parent_revision=revision,
|
||||
token=token,
|
||||
cache_dir=cache_dir,
|
||||
skip_dependencies=skip_dependencies,
|
||||
)
|
||||
|
||||
# before 4.x, pyannote.audio pipeline was using "version" key to
|
||||
@@ -242,10 +229,9 @@ class Pipeline(_Pipeline):
|
||||
config["dependencies"] = {"pyannote.audio": config["version"]}
|
||||
del config["version"]
|
||||
|
||||
if not skip_dependencies:
|
||||
# check that dependencies are available (in their required version)
|
||||
dependencies: dict[str, str] = config.get("dependencies", dict())
|
||||
check_dependencies(dependencies, "Pipeline")
|
||||
# check that dependencies are available (in their required version)
|
||||
dependencies: dict[str, str] = config.get("dependencies", dict())
|
||||
check_dependencies(dependencies, "Pipeline")
|
||||
|
||||
# initialize pipeline
|
||||
pipeline_name = config["pipeline"]["name"]
|
||||
|
||||
@@ -69,7 +69,6 @@ def get_model(
|
||||
model: PipelineModel,
|
||||
token: str | None = None,
|
||||
cache_dir: Path | str | None = None,
|
||||
skip_dependencies: bool = False,
|
||||
) -> Model:
|
||||
"""Load pretrained model and set it into `eval` mode.
|
||||
|
||||
@@ -84,9 +83,6 @@ def get_model(
|
||||
Huggingface token to be used for downloading from Huggingface hub.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where files downloaded from Huggingface hub are stored.
|
||||
skip_dependencies : bool, optional
|
||||
If True, skip dependency check. Defaults to False.
|
||||
Use at your own risk, as this may lead to unexpected behavior.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -115,7 +111,6 @@ def get_model(
|
||||
token=token,
|
||||
cache_dir=cache_dir,
|
||||
strict=False,
|
||||
skip_dependencies=skip_dependencies,
|
||||
)
|
||||
if _model:
|
||||
model = _model
|
||||
@@ -123,7 +118,6 @@ def get_model(
|
||||
elif isinstance(model, Mapping):
|
||||
model.setdefault("token", token)
|
||||
model.setdefault("cache_dir", cache_dir)
|
||||
model.setdefault("skip_dependencies", skip_dependencies)
|
||||
model = Model.from_pretrained(**model)
|
||||
|
||||
else:
|
||||
@@ -146,7 +140,6 @@ def get_calibration(
|
||||
calibration: PipelineCalibration,
|
||||
token: str | None = None,
|
||||
cache_dir: Path | str | None = None,
|
||||
skip_dependencies: bool = False,
|
||||
) -> Calibration | None:
|
||||
"""Load pretrained calibration
|
||||
|
||||
@@ -161,9 +154,6 @@ def get_calibration(
|
||||
Huggingface token to be used for downloading from Huggingface hub.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where files downloaded from Huggingface hub are stored.
|
||||
skip_dependencies : bool, optional
|
||||
If True, skip dependency check. Defaults to False.
|
||||
Use at your own risk, as this may lead to unexpected behavior.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -183,13 +173,11 @@ def get_calibration(
|
||||
calibration,
|
||||
token=token,
|
||||
cache_dir=cache_dir,
|
||||
skip_dependencies=skip_dependencies,
|
||||
)
|
||||
|
||||
elif isinstance(calibration, Dict):
|
||||
calibration.setdefault("token", token)
|
||||
calibration.setdefault("cache_dir", cache_dir)
|
||||
calibration.setdefault("skip_dependencies", skip_dependencies)
|
||||
loaded_calibration = Calibration.from_pretrained(**calibration)
|
||||
|
||||
else:
|
||||
@@ -208,7 +196,6 @@ def get_plda(
|
||||
plda: PipelinePLDA,
|
||||
token: str | None = None,
|
||||
cache_dir: Path | str | None = None,
|
||||
skip_dependencies: bool = False,
|
||||
) -> PLDA | None:
|
||||
"""Load pretrained calibration
|
||||
|
||||
@@ -223,9 +210,6 @@ def get_plda(
|
||||
Huggingface token to be used for downloading from Huggingface hub.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where files downloaded from Huggingface hub are stored.
|
||||
skip_dependencies : bool, optional
|
||||
If True, skip dependency check. Defaults to False.
|
||||
Use at your own risk, as this may lead to unexpected behavior.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -241,14 +225,11 @@ def get_plda(
|
||||
loaded_plda = plda
|
||||
|
||||
elif isinstance(plda, str):
|
||||
loaded_plda = PLDA.from_pretrained(
|
||||
plda, token=token, cache_dir=cache_dir, skip_dependencies=skip_dependencies
|
||||
)
|
||||
loaded_plda = PLDA.from_pretrained(plda, token=token, cache_dir=cache_dir)
|
||||
|
||||
elif isinstance(plda, Dict):
|
||||
plda.setdefault("token", token)
|
||||
plda.setdefault("cache_dir", cache_dir)
|
||||
plda.setdefault("skip_dependencies", skip_dependencies)
|
||||
loaded_plda = PLDA.from_pretrained(**plda)
|
||||
|
||||
else:
|
||||
|
||||
@@ -22,6 +22,8 @@
|
||||
# SOFTWARE.
|
||||
|
||||
import importlib.metadata
|
||||
import os
|
||||
import warnings
|
||||
from importlib.metadata import PackageNotFoundError
|
||||
|
||||
from packaging.version import InvalidVersion, Version
|
||||
@@ -32,7 +34,8 @@ class MissingDependency(Exception):
|
||||
|
||||
def __init__(self, what: str, dependency: str, required: Version) -> None:
|
||||
super().__init__(
|
||||
f"{what} requires {dependency} ~ {required} but it is not installed."
|
||||
f"{what} requires {dependency} ~ {required} but it is not installed. "
|
||||
"Use PYANNOTE_SKIP_DEPENDENCY_CHECK=1 to proceed anyway."
|
||||
)
|
||||
self.dependency = dependency
|
||||
self.required = required
|
||||
@@ -45,7 +48,8 @@ class WrongDependencyVersion(Exception):
|
||||
self, what: str, dependency: str, required: Version, available: Version
|
||||
) -> None:
|
||||
super().__init__(
|
||||
f"{what} requires {dependency} ~ {required} but {available} is installed."
|
||||
f"{what} requires {dependency} ~ {required} but {available} is installed. "
|
||||
"Use PYANNOTE_SKIP_DEPENDENCY_CHECK=1 to proceed anyway."
|
||||
)
|
||||
self.dependency = dependency
|
||||
self.required = required
|
||||
@@ -63,6 +67,17 @@ def check_dependencies(dependencies: dict[str, str], what: str) -> None:
|
||||
If a required dependency has an incompatible version.
|
||||
"""
|
||||
|
||||
skip_dependency_check: bool = os.getenv(
|
||||
"PYANNOTE_SKIP_DEPENDENCY_CHECK", False
|
||||
) in [
|
||||
True,
|
||||
"1",
|
||||
"true",
|
||||
"True",
|
||||
"yes",
|
||||
"Yes",
|
||||
]
|
||||
|
||||
for dependency, version in dependencies.items():
|
||||
required: Version = Version(version)
|
||||
|
||||
@@ -70,7 +85,14 @@ def check_dependencies(dependencies: dict[str, str], what: str) -> None:
|
||||
_version = importlib.metadata.version(dependency)
|
||||
|
||||
except PackageNotFoundError:
|
||||
raise MissingDependency(what, dependency, required)
|
||||
if skip_dependency_check:
|
||||
warnings.warn(
|
||||
f"{what} requires {dependency} ~ {required} but it is not installed. "
|
||||
"Proceeding anyway.",
|
||||
UserWarning,
|
||||
)
|
||||
else:
|
||||
raise MissingDependency(what, dependency, required)
|
||||
|
||||
_version = importlib.metadata.version(dependency)
|
||||
|
||||
@@ -78,7 +100,14 @@ def check_dependencies(dependencies: dict[str, str], what: str) -> None:
|
||||
available: Version = Version(_version)
|
||||
|
||||
except InvalidVersion:
|
||||
# if the version cannot be parsed, we take our chance and assume it is compatible
|
||||
# if the version cannot be parsed, we take our chance and assume it is compatible but warn
|
||||
# the user that something fishy is going on (unless they asked us to skip the check)
|
||||
if not skip_dependency_check:
|
||||
warnings.warn(
|
||||
f"{what} requires {dependency} ~ {required} but we could not figure out which version is installed. "
|
||||
"Proceeding anyway. Use PYANNOTE_SKIP_DEPENDENCY_CHECK=1 to remove this warning.",
|
||||
UserWarning,
|
||||
)
|
||||
continue
|
||||
|
||||
if available.major != required.major:
|
||||
@@ -88,5 +117,12 @@ def check_dependencies(dependencies: dict[str, str], what: str) -> None:
|
||||
continue
|
||||
|
||||
else:
|
||||
# for all other dependencies, we raise an error on major version mismatch
|
||||
raise WrongDependencyVersion(what, dependency, required, available)
|
||||
# for all other dependencies, we raise an error (or warn) on major version mismatch
|
||||
if skip_dependency_check:
|
||||
warnings.warn(
|
||||
f"{what} requires {dependency} ~ {required} but {available} is installed. "
|
||||
"Proceeding anyway.",
|
||||
UserWarning,
|
||||
)
|
||||
else:
|
||||
raise WrongDependencyVersion(what, dependency, required, available)
|
||||
Reference in new issue
Block a user