fix(task): fix Task.prepare_data to support saving preprocessors that produce int values in metadata (#1985)

This commit is contained in:
Han Han authored and GitHub committed 2026-02-20 14:49:20 +01:00
1 parent 3d22ecb2ab
commit 902a0099a7
2 files changed
+6 -10

No files matched your search

+4
View File
@@ -1,5 +1,9 @@
# CHANGELOG
## Version 4.0.5 (2026-02-20)
- fix(task): fix `Task.prepare_data` to support saving preprocessors that produce `int` values in metadata [@lylyhan](http://github.com/lylyhan)
## Version 4.0.4 (2026-02-07)
- feat(sample): add transcription of sample file
+2 -10
View File
@@ -274,7 +274,6 @@ class Task(lightning.LightningDataModule):
# metadata cache
self.cache = Path(cache) if cache else cache
# batching
self.duration = duration
self.min_duration = duration if min_duration is None else min_duration
@@ -331,7 +330,6 @@ class Task(lightning.LightningDataModule):
}
"""
if self.cache:
# check if cache exists and is not empty:
if self.cache.exists() and self.cache.stat().st_size > 0:
@@ -343,7 +341,6 @@ class Task(lightning.LightningDataModule):
# if no cache was provided by user, create a temporary file
# in system directory used for temp files
self.cache = Path(mkstemp()[1])
# list of possible values for each metadata key
# (will become .prepared_data[""])
metadata_unique_values = defaultdict(list)
@@ -417,6 +414,8 @@ class Task(lightning.LightningDataModule):
metadatum[key] = metadata_unique_values[key].index(value)
elif isinstance(value, int):
if value not in metadata_unique_values[key]:
metadata_unique_values[key].append(value)
metadatum[key] = value
else:
@@ -548,13 +547,10 @@ class Task(lightning.LightningDataModule):
("database_label_idx", get_dtype(max(a[4] for a in annotations))),
("global_label_idx", get_dtype(max(a[5] for a in annotations))),
]
# save all protocol data in a dict
prepared_data = {}
# keep track of protocol name
prepared_data["protocol"] = self.protocol.name
prepared_data["audio-path"] = np.array(audios, dtype=np.str_)
audios.clear()
@@ -568,7 +564,6 @@ class Task(lightning.LightningDataModule):
annotated_regions, dtype=region_dtype
)
annotated_regions.clear()
prepared_data["audio-regions-ids"] = np.array(
audio_regions_ids, dtype=[("start", "i"), ("end", "i")]
)
@@ -585,7 +580,6 @@ class Task(lightning.LightningDataModule):
audio_segments_ids.clear()
prepared_data["metadata-values"] = metadata_unique_values
for database, labels in database_unique_labels.items():
prepared_data[f"metadata-{database}-labels"] = np.array(
labels, dtype=np.str_
@@ -594,12 +588,10 @@ class Task(lightning.LightningDataModule):
prepared_data["metadata-labels"] = np.array(unique_labels, dtype=np.str_)
unique_labels.clear()
if self.has_validation:
self.prepare_validation(prepared_data)
self.post_prepare_data(prepared_data)
# save prepared data on the disk
with open(self.cache, "wb") as cache_file:
np.savez_compressed(cache_file, **prepared_data)