230 lines
7.3 KiB
Plaintext
Executable File
230 lines
7.3 KiB
Plaintext
Executable File
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from datasets import Dataset\n",
|
|
"import numpy as np\n",
|
|
"import torch\n",
|
|
"from lsync.config import TARGET_SR\n",
|
|
"import librosa\n",
|
|
"from transformers import Wav2Vec2Processor, Wav2Vec2FeatureExtractor, Wav2Vec2CTCTokenizer\n",
|
|
"\n",
|
|
"import evaluate\n",
|
|
"\n",
|
|
"metrics = evaluate.load(\"wer\")\n",
|
|
"MODEL_ID = \"facebook/wav2vec2-base\"\n",
|
|
"\n",
|
|
"tokenizer = Wav2Vec2CTCTokenizer(\"vocab.json\", unk_token=\"[UNK]\", pad_token=\"[PAD]\", word_delimiter_token=\"|\")\n",
|
|
"feature_extractor = Wav2Vec2FeatureExtractor(feature_size=1, sampling_rate=16000, padding_value=0.0, do_normalize=True, return_attention_mask=False)\n",
|
|
"processor = Wav2Vec2Processor(feature_extractor=feature_extractor, tokenizer=tokenizer)"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def audio_processing(fp):\n",
|
|
" y, sr = librosa.load(fp, sr=TARGET_SR)\n",
|
|
" yt, _ = librosa.effects.trim(y, top_db=30)\n",
|
|
" return yt\n",
|
|
"\n",
|
|
"\n",
|
|
"def data_preprocess(data):\n",
|
|
" audio = audio_processing(data['path'])\n",
|
|
" result = {}\n",
|
|
" result['input_values'] = processor(\n",
|
|
" audio, sampling_rate=TARGET_SR).input_values[0]\n",
|
|
" with processor.as_target_processor():\n",
|
|
" result[\"labels\"] = processor(data[\"text\"]).input_ids\n",
|
|
" return result\n",
|
|
"\n",
|
|
"\n",
|
|
"def prepare_dataset(batch):\n",
|
|
" audio = batch[\"path\"]\n",
|
|
"\n",
|
|
" # batched output is \"un-batched\" to ensure mapping is correct\n",
|
|
" batch[\"input_values\"] = processor(\n",
|
|
" audio[\"array\"], sampling_rate=audio[\"sampling_rate\"]).input_values[0]\n",
|
|
" batch[\"labels\"] = processor(batch[\"text\"]).input_ids\n",
|
|
" return batch\n",
|
|
"\n",
|
|
"def get_datasets():\n",
|
|
" dataset_csv_path = \"dataset.csv\"\n",
|
|
" dataset = Dataset.from_csv(dataset_csv_path)\n",
|
|
" dataset = dataset.filter(lambda x: x[\"text\"] != \"\")\n",
|
|
" dataset.cleanup_cache_files()\n",
|
|
" dataset = dataset.train_test_split(test_size=0.05, seed=41)\n",
|
|
" # smaller_dataset = dataset['test'].train_test_split(test_size=0.05)\n",
|
|
" train_dataset = dataset['train']\n",
|
|
" test_dataset = dataset['test']\n",
|
|
"\n",
|
|
" train_dataset = train_dataset.map(\n",
|
|
" data_preprocess,\n",
|
|
" num_proc=8\n",
|
|
" )\n",
|
|
" test_dataset = test_dataset.map(\n",
|
|
" data_preprocess,\n",
|
|
" num_proc=8\n",
|
|
" )\n",
|
|
" return (train_dataset, test_dataset)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"train_dataset, test_dataset = get_datasets()"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Train"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"@dataclass\n",
|
|
"class DataCollatorCTCWithPadding:\n",
|
|
" processor: Wav2Vec2Processor\n",
|
|
" padding: Union[bool, str] = True\n",
|
|
" max_length: Optional[int] = None\n",
|
|
" max_length_labels: Optional[int] = None\n",
|
|
" pad_to_multiple_of: Optional[int] = None\n",
|
|
" pad_to_multiple_of_labels: Optional[int] = None\n",
|
|
"\n",
|
|
" def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:\n",
|
|
" input_features = [{\"input_values\": feature[\"input_values\"]}\n",
|
|
" for feature in features]\n",
|
|
" label_features = [{\"input_ids\": feature[\"labels\"]}\n",
|
|
" for feature in features]\n",
|
|
"\n",
|
|
" batch = self.processor.pad(\n",
|
|
" input_features,\n",
|
|
" padding=self.padding,\n",
|
|
" max_length=self.max_length,\n",
|
|
" pad_to_multiple_of=self.pad_to_multiple_of,\n",
|
|
" return_tensors=\"pt\",\n",
|
|
" )\n",
|
|
" with self.processor.as_target_processor():\n",
|
|
" labels_batch = self.processor.pad(\n",
|
|
" label_features,\n",
|
|
" padding=self.padding,\n",
|
|
" max_length=self.max_length_labels,\n",
|
|
" pad_to_multiple_of=self.pad_to_multiple_of_labels,\n",
|
|
" return_tensors=\"pt\",\n",
|
|
" )\n",
|
|
"\n",
|
|
" labels = labels_batch[\"input_ids\"].masked_fill(\n",
|
|
" labels_batch.attention_mask.ne(1), -100)\n",
|
|
"\n",
|
|
" batch[\"labels\"] = labels\n",
|
|
"\n",
|
|
" return batch\n",
|
|
"\n",
|
|
"\n",
|
|
"def compute_metrics(pred):\n",
|
|
" pred_logits = pred.predictions\n",
|
|
" pred_ids = np.argmax(pred_logits, axis=-1)\n",
|
|
" pred.label_ids[pred.label_ids == -100] = processor.tokenizer.pad_token_id\n",
|
|
" pred_str = processor.batch_decode(pred_ids)\n",
|
|
" label_str = processor.batch_decode(pred.label_ids, group_tokens=False)\n",
|
|
" wer = metrics.compute(predictions=pred_str, references=label_str)\n",
|
|
" return {\"wer\": wer}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Train\n",
|
|
"data_collator = DataCollatorCTCWithPadding(\n",
|
|
" processor=processor, padding=True)\n",
|
|
"\n",
|
|
"model = Wav2Vec2ForCTC.from_pretrained(\n",
|
|
" MODEL_ID,\n",
|
|
" ctc_loss_reduction=\"mean\",\n",
|
|
" pad_token_id=processor.tokenizer.pad_token_id,\n",
|
|
")\n",
|
|
"model.freeze_feature_extractor()\n",
|
|
"torch.cuda.empty_cache()\n",
|
|
"training_args = TrainingArguments(\n",
|
|
" output_dir=\"model\",\n",
|
|
" per_device_train_batch_size=2,\n",
|
|
" per_device_eval_batch_size=2,\n",
|
|
" gradient_accumulation_steps=2,\n",
|
|
" num_train_epochs=5,\n",
|
|
" fp16=True,\n",
|
|
" save_steps=3000,\n",
|
|
" eval_steps=20000,\n",
|
|
" logging_steps=2000,\n",
|
|
" learning_rate=5e-4,\n",
|
|
" weight_decay=0.0001,\n",
|
|
" warmup_steps=1000,\n",
|
|
" save_total_limit=8,\n",
|
|
" evaluation_strategy=\"steps\",\n",
|
|
" # load_best_model_at_end=True,\n",
|
|
" # metric_for_best_model=\"eval_wer\"\n",
|
|
")\n",
|
|
"\n",
|
|
"trainer = Trainer(\n",
|
|
" model=model,\n",
|
|
" data_collator=data_collator,\n",
|
|
" args=training_args,\n",
|
|
" train_dataset=train_dataset,\n",
|
|
" eval_dataset=test_dataset,\n",
|
|
" tokenizer=processor.feature_extractor,\n",
|
|
")\n",
|
|
"\n",
|
|
"trainer.train(resume_from_checkpoint=True)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "lsync",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.12"
|
|
},
|
|
"orig_nbformat": 4
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|