Files
2026-03-03 14:52:19 +09:00

7.3 KiB
Executable File

In [ ]:
from datasets import Dataset
import numpy as np
import torch
from lsync.config import TARGET_SR
import librosa
from transformers import Wav2Vec2Processor, Wav2Vec2FeatureExtractor, Wav2Vec2CTCTokenizer

import evaluate

metrics = evaluate.load("wer")
MODEL_ID = "facebook/wav2vec2-base"

tokenizer = Wav2Vec2CTCTokenizer("vocab.json", unk_token="[UNK]", pad_token="[PAD]", word_delimiter_token="|")
feature_extractor = Wav2Vec2FeatureExtractor(feature_size=1, sampling_rate=16000, padding_value=0.0, do_normalize=True, return_attention_mask=False)
processor = Wav2Vec2Processor(feature_extractor=feature_extractor, tokenizer=tokenizer)

Dataset

In [ ]:
def audio_processing(fp):
    y, sr = librosa.load(fp, sr=TARGET_SR)
    yt, _ = librosa.effects.trim(y, top_db=30)
    return yt


def data_preprocess(data):
    audio = audio_processing(data['path'])
    result = {}
    result['input_values'] = processor(
        audio, sampling_rate=TARGET_SR).input_values[0]
    with processor.as_target_processor():
        result["labels"] = processor(data["text"]).input_ids
    return result


def prepare_dataset(batch):
    audio = batch["path"]

    # batched output is "un-batched" to ensure mapping is correct
    batch["input_values"] = processor(
        audio["array"], sampling_rate=audio["sampling_rate"]).input_values[0]
    batch["labels"] = processor(batch["text"]).input_ids
    return batch

def get_datasets():
    dataset_csv_path = "dataset.csv"
    dataset = Dataset.from_csv(dataset_csv_path)
    dataset = dataset.filter(lambda x: x["text"] != "")
    dataset.cleanup_cache_files()
    dataset = dataset.train_test_split(test_size=0.05, seed=41)
    # smaller_dataset = dataset['test'].train_test_split(test_size=0.05)
    train_dataset = dataset['train']
    test_dataset = dataset['test']

    train_dataset = train_dataset.map(
        data_preprocess,
        num_proc=8
    )
    test_dataset = test_dataset.map(
        data_preprocess,
        num_proc=8
    )
    return (train_dataset, test_dataset)
In [ ]:
train_dataset, test_dataset = get_datasets()

Train

In [ ]:
@dataclass
class DataCollatorCTCWithPadding:
    processor: Wav2Vec2Processor
    padding: Union[bool, str] = True
    max_length: Optional[int] = None
    max_length_labels: Optional[int] = None
    pad_to_multiple_of: Optional[int] = None
    pad_to_multiple_of_labels: Optional[int] = None

    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:
        input_features = [{"input_values": feature["input_values"]}
                          for feature in features]
        label_features = [{"input_ids": feature["labels"]}
                          for feature in features]

        batch = self.processor.pad(
            input_features,
            padding=self.padding,
            max_length=self.max_length,
            pad_to_multiple_of=self.pad_to_multiple_of,
            return_tensors="pt",
        )
        with self.processor.as_target_processor():
            labels_batch = self.processor.pad(
                label_features,
                padding=self.padding,
                max_length=self.max_length_labels,
                pad_to_multiple_of=self.pad_to_multiple_of_labels,
                return_tensors="pt",
            )

        labels = labels_batch["input_ids"].masked_fill(
            labels_batch.attention_mask.ne(1), -100)

        batch["labels"] = labels

        return batch


def compute_metrics(pred):
    pred_logits = pred.predictions
    pred_ids = np.argmax(pred_logits, axis=-1)
    pred.label_ids[pred.label_ids == -100] = processor.tokenizer.pad_token_id
    pred_str = processor.batch_decode(pred_ids)
    label_str = processor.batch_decode(pred.label_ids, group_tokens=False)
    wer = metrics.compute(predictions=pred_str, references=label_str)
    return {"wer": wer}
In [ ]:
# Train
data_collator = DataCollatorCTCWithPadding(
    processor=processor, padding=True)

model = Wav2Vec2ForCTC.from_pretrained(
    MODEL_ID,
    ctc_loss_reduction="mean",
    pad_token_id=processor.tokenizer.pad_token_id,
)
model.freeze_feature_extractor()
torch.cuda.empty_cache()
training_args = TrainingArguments(
    output_dir="model",
    per_device_train_batch_size=2,
    per_device_eval_batch_size=2,
    gradient_accumulation_steps=2,
    num_train_epochs=5,
    fp16=True,
    save_steps=3000,
    eval_steps=20000,
    logging_steps=2000,
    learning_rate=5e-4,
    weight_decay=0.0001,
    warmup_steps=1000,
    save_total_limit=8,
    evaluation_strategy="steps",
    # load_best_model_at_end=True,
    # metric_for_best_model="eval_wer"
)

trainer = Trainer(
    model=model,
    data_collator=data_collator,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=test_dataset,
    tokenizer=processor.feature_extractor,
)

trainer.train(resume_from_checkpoint=True)