fix: fix support for non-ASCII characters

Fixes #1608 #1612 

Co-authored-by: Hervé BREDIN <hbredin@users.noreply.github.com>
This commit is contained in:
Clément PagésandHervé BREDIN authored and GitHub committed 2024-01-15 13:11:21 +01:00
1 parent 9e4ec5f6e0
commit b41b176e3f
7 files changed
+24 -24

No files matched your search

+6 -6
View File
@@ -215,11 +215,11 @@ class Task(pl.LightningDataModule):
pyannote.database protocol
cache : str, optional
As (meta-)data preparation might take a very long time for large datasets,
it can be cached to disk for later (and faster!) re-use.
it can be cached to disk for later (and faster!) re-use.
When `cache` does not exist, `Task.prepare_data()` generates training
and validation metadata from `protocol` and save them to disk.
When `cache` exists, `Task.prepare_data()` is skipped and (meta)-data
are loaded from disk. Defaults to a temporary path.
are loaded from disk. Defaults to a temporary path.
duration : float, optional
Chunks duration in seconds. Defaults to two seconds (2.).
min_duration : float, optional
@@ -567,7 +567,7 @@ class Task(pl.LightningDataModule):
# keep track of protocol name
prepared_data["protocol"] = self.protocol.name
prepared_data["audio-path"] = np.array(audios, dtype=np.string_)
prepared_data["audio-path"] = np.array(audios, dtype=np.str_)
audios.clear()
prepared_data["audio-metadata"] = np.array(metadata, dtype=metadata_dtype)
@@ -576,7 +576,7 @@ class Task(pl.LightningDataModule):
prepared_data["audio-info"] = np.array(audio_infos, dtype=info_dtype)
audio_infos.clear()
prepared_data["audio-encoding"] = np.array(audio_encodings, dtype=np.string_)
prepared_data["audio-encoding"] = np.array(audio_encodings, dtype=np.str_)
audio_encodings.clear()
prepared_data["audio-annotated"] = np.array(annotated_duration)
@@ -596,11 +596,11 @@ class Task(pl.LightningDataModule):
for database, labels in database_unique_labels.items():
prepared_data[f"metadata-{database}-labels"] = np.array(
labels, dtype=np.string_
labels, dtype=np.str_
)
database_unique_labels.clear()
prepared_data["metadata-labels"] = np.array(unique_labels, dtype=np.string_)
prepared_data["metadata-labels"] = np.array(unique_labels, dtype=np.str_)
unique_labels.clear()
self.prepare_validation(prepared_data)
+2 -3
View File
@@ -48,16 +48,15 @@ class SegmentationTask(Task):
def get_file(self, file_id):
file = dict()
file["audio"] = str(self.prepared_data["audio-path"][file_id], encoding="utf-8")
file["audio"] = self.prepared_data["audio-path"][file_id]
_audio_info = self.prepared_data["audio-info"][file_id]
_encoding = self.prepared_data["audio-encoding"][file_id]
encoding = self.prepared_data["audio-encoding"][file_id]
sample_rate = _audio_info["sample_rate"]
num_frames = _audio_info["num_frames"]
num_channels = _audio_info["num_channels"]
bits_per_sample = _audio_info["bits_per_sample"]
encoding = str(_encoding, encoding="utf-8")
file["torchaudio.info"] = AudioMetaData(
sample_rate=sample_rate,
num_frames=num_frames,
@@ -49,13 +49,13 @@ class MultiLabelSegmentation(SegmentationTask):
Parameters
----------
protocol : Protocol
cache : str, optional
cache : str, optional
As (meta-)data preparation might take a very long time for large datasets,
it can be cached to disk for later (and faster!) re-use.
it can be cached to disk for later (and faster!) re-use.
When `cache` does not exist, `Task.prepare_data()` generates training
and validation metadata from `protocol` and save them to disk.
When `cache` exists, `Task.prepare_data()` is skipped and (meta)-data
are loaded from disk. Defaults to a temporary path.
are loaded from disk. Defaults to a temporary path.
classes : List[str], optional
List of classes. Defaults to the list of classes available in the training set.
duration : float, optional
@@ -179,7 +179,7 @@ class MultiLabelSegmentation(SegmentationTask):
[classes.index(klass) for klass in file_classes]
)
prepared_data["classes-list"] = np.array(classes, dtype=np.string_)
prepared_data["classes-list"] = np.array(classes, dtype=np.str_)
self.classes = classes
else:
@@ -216,7 +216,7 @@ class MultiLabelSegmentation(SegmentationTask):
]
)
prepared_data["classes-list"] = np.array(self.classes, dtype=np.string_)
prepared_data["classes-list"] = np.array(self.classes, dtype=np.str_)
# convert annotated_classes (which is a list of list of classes, one list of classes per file)
# into a single (num_files x num_classes) numpy array:
+1
View File
@@ -2,6 +2,7 @@ Protocols:
Debug:
SpeakerDiarization:
Debug:
scope: database
train:
uri: debug.train.lst
annotation: debug.train.rttm
+1 -1
View File
@@ -1,4 +1,4 @@
trn00
trñ00
trn01
trn02
trn03
+9 -9
View File
@@ -1,26 +1,26 @@
SPEAKER trn00 1 3.168 0.800 <NA> <NA> MEO069 <NA> <NA>
SPEAKER trn00 1 5.463 0.640 <NA> <NA> MEO069 <NA> <NA>
SPEAKER trn00 1 3.168 0.800 <NA> <NA> MÉO069 <NA> <NA>
SPEAKER trn00 1 5.463 0.640 <NA> <NA> MÉO069 <NA> <NA>
SPEAKER trn00 1 5.496 0.574 <NA> <NA> MEE068 <NA> <NA>
SPEAKER trn00 1 10.454 0.499 <NA> <NA> MEO069 <NA> <NA>
SPEAKER trn00 1 10.454 0.499 <NA> <NA> MÉO069 <NA> <NA>
SPEAKER trn00 1 11.040 4.592 <NA> <NA> MEE068 <NA> <NA>
SPEAKER trn00 1 16.736 1.410 <NA> <NA> MEO069 <NA> <NA>
SPEAKER trn00 1 16.736 1.410 <NA> <NA> MÉO069 <NA> <NA>
SPEAKER trn00 1 16.980 2.778 <NA> <NA> MEE067 <NA> <NA>
SPEAKER trn00 1 18.883 0.490 <NA> <NA> MEE068 <NA> <NA>
SPEAKER trn00 1 18.985 1.831 <NA> <NA> MEO069 <NA> <NA>
SPEAKER trn00 1 18.985 1.831 <NA> <NA> MÉO069 <NA> <NA>
SPEAKER trn00 1 20.944 0.447 <NA> <NA> MEE067 <NA> <NA>
SPEAKER trn00 1 21.392 4.465 <NA> <NA> MEE068 <NA> <NA>
SPEAKER trn00 1 22.928 0.384 <NA> <NA> MEO069 <NA> <NA>
SPEAKER trn00 1 25.001 2.471 <NA> <NA> MEO069 <NA> <NA>
SPEAKER trn00 1 22.928 0.384 <NA> <NA> MÉO069 <NA> <NA>
SPEAKER trn00 1 25.001 2.471 <NA> <NA> MÉO069 <NA> <NA>
SPEAKER trn00 1 28.033 1.967 <NA> <NA> MEE068 <NA> <NA>
SPEAKER trn01 1 2.977 0.391 <NA> <NA> FEO066 <NA> <NA>
SPEAKER trn01 1 18.705 0.964 <NA> <NA> MEE068 <NA> <NA>
SPEAKER trn01 1 22.269 0.457 <NA> <NA> FEO065 <NA> <NA>
SPEAKER trn01 1 28.474 1.526 <NA> <NA> MEO069 <NA> <NA>
SPEAKER trn01 1 28.474 1.526 <NA> <NA> MÉO069 <NA> <NA>
SPEAKER trn01 1 28.593 1.407 <NA> <NA> FEO066 <NA> <NA>
SPEAKER trn01 1 28.993 1.007 <NA> <NA> FEO065 <NA> <NA>
SPEAKER trn02 1 20.704 0.688 <NA> <NA> FEO066 <NA> <NA>
SPEAKER trn03 1 0.000 1.184 <NA> <NA> MEE067 <NA> <NA>
SPEAKER trn03 1 1.104 28.896 <NA> <NA> MEO069 <NA> <NA>
SPEAKER trn03 1 1.104 28.896 <NA> <NA> MÉO069 <NA> <NA>
SPEAKER trn04 1 14.032 1.744 <NA> <NA> MEE076 <NA> <NA>
SPEAKER trn04 1 14.345 2.471 <NA> <NA> MEO074 <NA> <NA>
SPEAKER trn04 1 16.736 7.216 <NA> <NA> MEE075 <NA> <NA>
File renamed without changes.