feat: add option to skip dependency check (#1870)

This commit is contained in:
Hervé BREDIN authored and GitHub committed 2025-05-20 21:19:57 +02:00
1 parent 0fa7e086b6
commit 5e56f06866
4 files changed
+53 -11

No files matched your search

+1
View File
@@ -64,6 +64,7 @@ Models can now be stored alongside their pipelines in the same repository, strea
- improve(setup): switch to `uv`
- improve(setup): switch to `lightning` from `pytorch-lightning`
- improve(utils): improve dependency check when loading pretrained models and/or pipeline
- improve(utils): add option to skip dependency check
### Fixes
+8 -3
View File
@@ -505,6 +505,7 @@ class Model(lightning.LightningModule):
subfolder: Optional[str] = None,
token: Union[str, bool, None] = None,
cache_dir: Union[Path, str, None] = None,
skip_dependencies: bool = False,
**kwargs,
) -> Optional["Model"]:
"""Load pretrained model
@@ -531,6 +532,9 @@ class Model(lightning.LightningModule):
kwargs: optional
Any extra keyword args needed to init the model.
Can also be used to override saved hyperparameter values.
skip_dependencies : bool, optional
If True, skip dependency check. Defaults to False.
Use at your own risk, as this may lead to unexpected behavior.
Returns
-------
@@ -578,9 +582,10 @@ class Model(lightning.LightningModule):
# load checkpoint using lightning
loaded_checkpoint = pl_load(path_to_model_checkpoint, map_location=map_location)
# check that the checkpoint is compatible with the current version
versions = loaded_checkpoint["pyannote.audio"]["versions"]
check_dependencies({"pyannote.audio": versions["pyannote.audio"]}, "Model")
if not skip_dependencies:
# check that the checkpoint is compatible with the current version
versions = loaded_checkpoint["pyannote.audio"]["versions"]
check_dependencies({"pyannote.audio": versions["pyannote.audio"]}, "Model")
# obtain model class from the checkpoint
module_name: str = loaded_checkpoint["pyannote.audio"]["architecture"]["module"]
+32 -6
View File
@@ -51,6 +51,7 @@ def expand_subfolders(
revision: Optional[Text] = None,
cache_dir: Union[Path, str, None] = None,
token: Optional[Text] = None,
skip_dependencies: bool = False,
) -> None:
"""Expand $model subfolders in config
@@ -70,6 +71,9 @@ def expand_subfolders(
Huggingface token to be used for downloading from Huggingface hub.
cache_dir: Path or str, optional
Path to the folder where files downloaded from Huggingface hub are stored.
skip_dependencies : bool, optional
If True, skip dependency check. Defaults to False.
Use at your own risk, as this may lead to unexpected behavior.
"""
if isinstance(config, dict):
@@ -82,10 +86,16 @@ def expand_subfolders(
"subfolder": subfolder,
"token": token,
"cache_dir": cache_dir,
"skip_dependencies": skip_dependencies,
}
else:
expand_subfolders(
value, model_id, revision=revision, token=token, cache_dir=cache_dir
value,
model_id,
revision=revision,
token=token,
cache_dir=cache_dir,
skip_dependencies=skip_dependencies,
)
elif isinstance(config, list):
@@ -98,10 +108,16 @@ def expand_subfolders(
"subfolder": subfolder,
"token": token,
"cache_dir": cache_dir,
"skip_dependencies": skip_dependencies,
}
else:
expand_subfolders(
value, model_id, revision=revision, token=token, cache_dir=cache_dir
value,
model_id,
revision=revision,
token=token,
cache_dir=cache_dir,
skip_dependencies=skip_dependencies,
)
@@ -113,6 +129,7 @@ class Pipeline(_Pipeline):
hparams_file: Union[Text, Path] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
skip_dependencies: bool = False,
) -> Optional["Pipeline"]:
"""Load pretrained pipeline
@@ -128,6 +145,9 @@ class Pipeline(_Pipeline):
Token to be used for the download.
cache_dir: Path or str, optional
Path to the folder where cached files are stored.
skip_dependencies : bool, optional
If True, skip dependency check. Defaults to False.
Use at your own risk, as this may lead to unexpected behavior.
"""
# if checkpoint is a directory, look for the pipeline checkpoint
@@ -159,7 +179,12 @@ class Pipeline(_Pipeline):
# expand $model/{subfolder}-like entries in config
expand_subfolders(
config, model_id, revision=revision, token=token, cache_dir=cache_dir
config,
model_id,
revision=revision,
token=token,
cache_dir=cache_dir,
skip_dependencies=skip_dependencies,
)
# before 4.x, pyannote.audio pipeline was using "version" key to
@@ -168,9 +193,10 @@ class Pipeline(_Pipeline):
config["dependencies"] = {"pyannote.audio": config["version"]}
del config["version"]
# check that dependencies are available (in their required version)
dependencies: dict[str, str] = config.get("dependencies", dict())
check_dependencies(dependencies, "Pipeline")
if not skip_dependencies:
# check that dependencies are available (in their required version)
dependencies: dict[str, str] = config.get("dependencies", dict())
check_dependencies(dependencies, "Pipeline")
# initialize pipeline
pipeline_name = config["pipeline"]["name"]
+12 -2
View File
@@ -38,6 +38,7 @@ def get_model(
model: PipelineModel,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
skip_dependencies: bool = False,
) -> Model:
"""Load pretrained model and set it into `eval` mode.
@@ -52,6 +53,9 @@ def get_model(
Huggingface token to be used for downloading from Huggingface hub.
cache_dir: Path or str, optional
Path to the folder where files downloaded from Huggingface hub are stored.
skip_dependencies : bool, optional
If True, skip dependency check. Defaults to False.
Use at your own risk, as this may lead to unexpected behavior.
Returns
-------
@@ -76,7 +80,7 @@ def get_model(
elif isinstance(model, Text):
_model = Model.from_pretrained(
model, token=token, cache_dir=cache_dir, strict=False
model, token=token, cache_dir=cache_dir, strict=False, skip_dependencies=skip_dependencies
)
if _model:
model = _model
@@ -84,6 +88,7 @@ def get_model(
elif isinstance(model, Mapping):
model.setdefault("token", token)
model.setdefault("cache_dir", cache_dir)
model.setdefault("skip_dependencies", skip_dependencies)
model = Model.from_pretrained(**model)
else:
@@ -106,6 +111,7 @@ def get_calibration(
calibration: PipelineCalibration,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
skip_dependencies: bool = False,
) -> Optional[Calibration]:
"""Load pretrained calibration
@@ -120,6 +126,9 @@ def get_calibration(
Huggingface token to be used for downloading from Huggingface hub.
cache_dir: Path or str, optional
Path to the folder where files downloaded from Huggingface hub are stored.
skip_dependencies : bool, optional
If True, skip dependency check. Defaults to False.
Use at your own risk, as this may lead to unexpected behavior.
Returns
-------
@@ -136,12 +145,13 @@ def get_calibration(
elif isinstance(calibration, Text):
loaded_calibration = Calibration.from_pretrained(
calibration, token=token, cache_dir=cache_dir
calibration, token=token, cache_dir=cache_dir, skip_dependencies=skip_dependencies
)
elif isinstance(calibration, Dict):
calibration.setdefault("token", token)
calibration.setdefault("cache_dir", cache_dir)
calibration.setdefault("skip_dependencies", skip_dependencies)
loaded_calibration = Calibration.from_pretrained(**calibration)
else: