{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from datasets import Dataset\n", "import numpy as np\n", "import torch\n", "from lsync.config import TARGET_SR\n", "import librosa\n", "from transformers import Wav2Vec2Processor, Wav2Vec2FeatureExtractor, Wav2Vec2CTCTokenizer\n", "\n", "import evaluate\n", "\n", "metrics = evaluate.load(\"wer\")\n", "MODEL_ID = \"facebook/wav2vec2-base\"\n", "\n", "tokenizer = Wav2Vec2CTCTokenizer(\"vocab.json\", unk_token=\"[UNK]\", pad_token=\"[PAD]\", word_delimiter_token=\"|\")\n", "feature_extractor = Wav2Vec2FeatureExtractor(feature_size=1, sampling_rate=16000, padding_value=0.0, do_normalize=True, return_attention_mask=False)\n", "processor = Wav2Vec2Processor(feature_extractor=feature_extractor, tokenizer=tokenizer)" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### Dataset" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def audio_processing(fp):\n", " y, sr = librosa.load(fp, sr=TARGET_SR)\n", " yt, _ = librosa.effects.trim(y, top_db=30)\n", " return yt\n", "\n", "\n", "def data_preprocess(data):\n", " audio = audio_processing(data['path'])\n", " result = {}\n", " result['input_values'] = processor(\n", " audio, sampling_rate=TARGET_SR).input_values[0]\n", " with processor.as_target_processor():\n", " result[\"labels\"] = processor(data[\"text\"]).input_ids\n", " return result\n", "\n", "\n", "def prepare_dataset(batch):\n", " audio = batch[\"path\"]\n", "\n", " # batched output is \"un-batched\" to ensure mapping is correct\n", " batch[\"input_values\"] = processor(\n", " audio[\"array\"], sampling_rate=audio[\"sampling_rate\"]).input_values[0]\n", " batch[\"labels\"] = processor(batch[\"text\"]).input_ids\n", " return batch\n", "\n", "def get_datasets():\n", " dataset_csv_path = \"dataset.csv\"\n", " dataset = Dataset.from_csv(dataset_csv_path)\n", " dataset = dataset.filter(lambda x: x[\"text\"] != \"\")\n", " dataset.cleanup_cache_files()\n", " dataset = dataset.train_test_split(test_size=0.05, seed=41)\n", " # smaller_dataset = dataset['test'].train_test_split(test_size=0.05)\n", " train_dataset = dataset['train']\n", " test_dataset = dataset['test']\n", "\n", " train_dataset = train_dataset.map(\n", " data_preprocess,\n", " num_proc=8\n", " )\n", " test_dataset = test_dataset.map(\n", " data_preprocess,\n", " num_proc=8\n", " )\n", " return (train_dataset, test_dataset)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "train_dataset, test_dataset = get_datasets()" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### Train" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "@dataclass\n", "class DataCollatorCTCWithPadding:\n", " processor: Wav2Vec2Processor\n", " padding: Union[bool, str] = True\n", " max_length: Optional[int] = None\n", " max_length_labels: Optional[int] = None\n", " pad_to_multiple_of: Optional[int] = None\n", " pad_to_multiple_of_labels: Optional[int] = None\n", "\n", " def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:\n", " input_features = [{\"input_values\": feature[\"input_values\"]}\n", " for feature in features]\n", " label_features = [{\"input_ids\": feature[\"labels\"]}\n", " for feature in features]\n", "\n", " batch = self.processor.pad(\n", " input_features,\n", " padding=self.padding,\n", " max_length=self.max_length,\n", " pad_to_multiple_of=self.pad_to_multiple_of,\n", " return_tensors=\"pt\",\n", " )\n", " with self.processor.as_target_processor():\n", " labels_batch = self.processor.pad(\n", " label_features,\n", " padding=self.padding,\n", " max_length=self.max_length_labels,\n", " pad_to_multiple_of=self.pad_to_multiple_of_labels,\n", " return_tensors=\"pt\",\n", " )\n", "\n", " labels = labels_batch[\"input_ids\"].masked_fill(\n", " labels_batch.attention_mask.ne(1), -100)\n", "\n", " batch[\"labels\"] = labels\n", "\n", " return batch\n", "\n", "\n", "def compute_metrics(pred):\n", " pred_logits = pred.predictions\n", " pred_ids = np.argmax(pred_logits, axis=-1)\n", " pred.label_ids[pred.label_ids == -100] = processor.tokenizer.pad_token_id\n", " pred_str = processor.batch_decode(pred_ids)\n", " label_str = processor.batch_decode(pred.label_ids, group_tokens=False)\n", " wer = metrics.compute(predictions=pred_str, references=label_str)\n", " return {\"wer\": wer}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Train\n", "data_collator = DataCollatorCTCWithPadding(\n", " processor=processor, padding=True)\n", "\n", "model = Wav2Vec2ForCTC.from_pretrained(\n", " MODEL_ID,\n", " ctc_loss_reduction=\"mean\",\n", " pad_token_id=processor.tokenizer.pad_token_id,\n", ")\n", "model.freeze_feature_extractor()\n", "torch.cuda.empty_cache()\n", "training_args = TrainingArguments(\n", " output_dir=\"model\",\n", " per_device_train_batch_size=2,\n", " per_device_eval_batch_size=2,\n", " gradient_accumulation_steps=2,\n", " num_train_epochs=5,\n", " fp16=True,\n", " save_steps=3000,\n", " eval_steps=20000,\n", " logging_steps=2000,\n", " learning_rate=5e-4,\n", " weight_decay=0.0001,\n", " warmup_steps=1000,\n", " save_total_limit=8,\n", " evaluation_strategy=\"steps\",\n", " # load_best_model_at_end=True,\n", " # metric_for_best_model=\"eval_wer\"\n", ")\n", "\n", "trainer = Trainer(\n", " model=model,\n", " data_collator=data_collator,\n", " args=training_args,\n", " train_dataset=train_dataset,\n", " eval_dataset=test_dataset,\n", " tokenizer=processor.feature_extractor,\n", ")\n", "\n", "trainer.train(resume_from_checkpoint=True)" ] } ], "metadata": { "kernelspec": { "display_name": "lsync", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.12" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }