Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c266ac5459 | ||
|
|
95fecb91c8 | ||
|
|
b1c8ac7de6 | ||
|
|
bf150e442e | ||
|
|
ed13dc8c6c | ||
|
|
c7d31883bc | ||
|
|
83afb81ac7 | ||
|
|
2d9ce44329 | ||
|
|
f4261f34e9 | ||
|
|
429658d4cc | ||
|
|
e0833da5dc |
No files matched your search
@@ -0,0 +1 @@
|
||||
3.10
|
||||
@@ -1,5 +1,10 @@
|
||||
<h1 align="center">WhisperX</h1>
|
||||
|
||||
## Recall.ai - Meeting Transcription API
|
||||
|
||||
If you’re looking for a transcription API for meetings, consider checking out [Recall.ai's Meeting Transcription API](https://www.recall.ai/product/meeting-transcription-api?utm_source=github&utm_medium=sponsorship&utm_campaign=mbain-whisperx), an API that works with Zoom, Google Meet, Microsoft Teams, and more. Recall.ai diarizes by pulling the speaker data and separate audio streams from the meeting platforms, which means 100% accurate speaker diarization with actual speaker names.
|
||||
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/m-bain/whisperX/stargazers">
|
||||
<img src="https://img.shields.io/github/stars/m-bain/whisperX.svg?colorA=orange&colorB=orange&logo=github"
|
||||
@@ -57,6 +62,15 @@ This repository provides fast automatic speech recognition (70x realtime with la
|
||||
|
||||
<h2 align="left" id="setup">Setup ⚙️</h2>
|
||||
|
||||
### 0. CUDA Installation
|
||||
|
||||
To use WhisperX with GPU acceleration, install the CUDA toolkit 12.8 before WhisperX. Skip this step if using only the CPU.
|
||||
|
||||
- For **Linux** users, install the CUDA toolkit 12.8 following this guide:
|
||||
[CUDA Installation Guide for Linux](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/).
|
||||
- For **Windows** users, download and install the CUDA toolkit 12.8:
|
||||
[CUDA Downloads](https://developer.nvidia.com/cuda-12-8-1-download-archive).
|
||||
|
||||
### 1. Simple Installation (Recommended)
|
||||
|
||||
The easiest way to install WhisperX is through PyPi:
|
||||
@@ -97,25 +111,6 @@ uv sync --all-extras --dev
|
||||
|
||||
You may also need to install ffmpeg, rust etc. Follow openAI instructions here https://github.com/openai/whisper#setup.
|
||||
|
||||
### Common Issues & Troubleshooting 🔧
|
||||
|
||||
#### libcudnn Dependencies (GPU Users)
|
||||
|
||||
If you're using WhisperX with GPU support and encounter errors like:
|
||||
|
||||
- `Could not load library libcudnn_ops_infer.so.8`
|
||||
- `Unable to load any of {libcudnn_cnn.so.9.1.0, libcudnn_cnn.so.9.1, libcudnn_cnn.so.9, libcudnn_cnn.so}`
|
||||
- `libcudnn_ops_infer.so.8: cannot open shared object file: No such file or directory`
|
||||
|
||||
This means your system is missing the CUDA Deep Neural Network library (cuDNN). This library is needed for GPU acceleration but isn't always installed by default.
|
||||
|
||||
**Install cuDNN (example for apt based systems):**
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install libcudnn8 libcudnn8-dev -y
|
||||
```
|
||||
|
||||
### Speaker Diarization
|
||||
|
||||
To **enable Speaker Diarization**, include your Hugging Face access token (read) that you can generate from [Here](https://huggingface.co/settings/tokens) after the `--hf_token` argument and accept the user agreement for the following models: [Segmentation](https://huggingface.co/pyannote/segmentation-3.0) and [Speaker-Diarization-3.1](https://huggingface.co/pyannote/speaker-diarization-3.1) (if you choose to use Speaker-Diarization 2.x, follow requirements [here](https://huggingface.co/pyannote/speaker-diarization) instead.)
|
||||
|
||||
+36
-8
@@ -2,23 +2,26 @@
|
||||
urls = { repository = "https://github.com/m-bain/whisperx" }
|
||||
authors = [{ name = "Max Bain" }]
|
||||
name = "whisperx"
|
||||
version = "3.4.1"
|
||||
version = "3.5.0"
|
||||
description = "Time-Accurate Automatic Speech Recognition using Whisper."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9, <3.13"
|
||||
license = { text = "BSD-2-Clause" }
|
||||
|
||||
dependencies = [
|
||||
"ctranslate2<4.5.0",
|
||||
"ctranslate2>=4.5.0",
|
||||
"faster-whisper>=1.1.1",
|
||||
"nltk>=3.9.1",
|
||||
"numpy>=2.0.2",
|
||||
"onnxruntime>=1.19",
|
||||
"pandas>=2.2.3",
|
||||
"pyannote-audio>=3.3.2",
|
||||
"torch>=2.5.1",
|
||||
"torchaudio>=2.5.1",
|
||||
# Restrict numpy, onnxruntime, pandas, av to be compatible with Python 3.9
|
||||
"numpy>=2.0.2,<2.1.0",
|
||||
"onnxruntime>=1.19,<1.20.0",
|
||||
"pandas>=2.2.3,<2.3.0",
|
||||
"av<16.0.0",
|
||||
"pyannote-audio>=3.3.2,<4.0.0",
|
||||
"torch>=2.7.1",
|
||||
"torchaudio",
|
||||
"transformers>=4.48.0",
|
||||
"triton>=3.3.0; sys_platform == 'linux'" # only install triton on Linux
|
||||
]
|
||||
|
||||
|
||||
@@ -34,3 +37,28 @@ include-package-data = true
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["whisperx*"]
|
||||
|
||||
[tool.uv.sources]
|
||||
torch = [
|
||||
{ index = "pytorch-cpu", marker = "sys_platform == 'darwin'" },
|
||||
{ index = "pytorch-cpu", marker = "platform_machine != 'x86_64' and sys_platform != 'darwin'" },
|
||||
{ index = "pytorch", marker = "platform_machine == 'x86_64' and sys_platform != 'darwin'" },
|
||||
]
|
||||
torchaudio = [
|
||||
{ index = "pytorch-cpu", marker = "sys_platform == 'darwin'" },
|
||||
{ index = "pytorch-cpu", marker = "platform_machine != 'x86_64' and sys_platform != 'darwin'" },
|
||||
{ index = "pytorch", marker = "platform_machine == 'x86_64' and sys_platform != 'darwin'" },
|
||||
]
|
||||
triton = [
|
||||
{ index = "pytorch", marker = "sys_platform == 'linux'" },
|
||||
]
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch"
|
||||
url = "https://download.pytorch.org/whl/cu128"
|
||||
explicit = true
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cpu"
|
||||
url = "https://download.pytorch.org/whl/cpu"
|
||||
explicit = true
|
||||
+21
-20
@@ -22,9 +22,8 @@ from whisperx.types import (
|
||||
SingleWordSegment,
|
||||
SegmentData,
|
||||
)
|
||||
from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktParameters
|
||||
|
||||
PUNKT_ABBREVIATIONS = ['dr', 'vs', 'mr', 'mrs', 'prof']
|
||||
import nltk
|
||||
from nltk.data import load as nltk_load
|
||||
|
||||
LANGUAGES_WITHOUT_SPACES = ["ja", "zh"]
|
||||
|
||||
@@ -53,7 +52,7 @@ DEFAULT_ALIGN_MODELS_HF = {
|
||||
"tr": "mpoyraz/wav2vec2-xls-r-300m-cv7-turkish",
|
||||
"da": "saattrupdan/wav2vec2-xls-r-300m-ftspeech",
|
||||
"he": "imvladikon/wav2vec2-xls-r-300m-hebrew",
|
||||
"vi": 'nguyenvulebinh/wav2vec2-base-vi',
|
||||
"vi": 'nguyenvulebinh/wav2vec2-base-vi-vlsp2020',
|
||||
"ko": "kresnik/wav2vec2-large-xlsr-korean",
|
||||
"ur": "kingabzpro/wav2vec2-large-xls-r-300m-Urdu",
|
||||
"te": "anuragshas/wav2vec2-large-xlsr-53-telugu",
|
||||
@@ -124,14 +123,14 @@ def align(
|
||||
"""
|
||||
Align phoneme recognition predictions to known transcription.
|
||||
"""
|
||||
|
||||
|
||||
if not torch.is_tensor(audio):
|
||||
if isinstance(audio, str):
|
||||
audio = load_audio(audio)
|
||||
audio = torch.from_numpy(audio)
|
||||
if len(audio.shape) == 1:
|
||||
audio = audio.unsqueeze(0)
|
||||
|
||||
|
||||
MAX_DURATION = audio.shape[1] / SAMPLE_RATE
|
||||
|
||||
model_dictionary = align_model_metadata["dictionary"]
|
||||
@@ -148,7 +147,7 @@ def align(
|
||||
base_progress = ((sdx + 1) / total_segments) * 100
|
||||
percent_complete = (50 + base_progress / 2) if combined_progress else base_progress
|
||||
print(f"Progress: {percent_complete:.2f}%...")
|
||||
|
||||
|
||||
num_leading = len(segment["text"]) - len(segment["text"].lstrip())
|
||||
num_trailing = len(segment["text"]) - len(segment["text"].rstrip())
|
||||
text = segment["text"]
|
||||
@@ -165,7 +164,7 @@ def align(
|
||||
# wav2vec2 models use "|" character to represent spaces
|
||||
if model_lang not in LANGUAGES_WITHOUT_SPACES:
|
||||
char_ = char_.replace(" ", "|")
|
||||
|
||||
|
||||
# ignore whitespace at beginning and end of transcript
|
||||
if cdx < num_leading:
|
||||
pass
|
||||
@@ -187,10 +186,12 @@ def align(
|
||||
# index for placeholder
|
||||
clean_wdx.append(wdx)
|
||||
|
||||
|
||||
punkt_param = PunktParameters()
|
||||
punkt_param.abbrev_types = set(PUNKT_ABBREVIATIONS)
|
||||
sentence_splitter = PunktSentenceTokenizer(punkt_param)
|
||||
|
||||
try:
|
||||
sentence_splitter = nltk_load('tokenizers/punkt/english.pickle')
|
||||
except LookupError:
|
||||
nltk.download('punkt_tab', quiet=True)
|
||||
sentence_splitter = nltk_load('tokenizers/punkt/english.pickle')
|
||||
sentence_spans = list(sentence_splitter.span_tokenize(text))
|
||||
|
||||
segment_data[sdx] = {
|
||||
@@ -199,12 +200,12 @@ def align(
|
||||
"clean_wdx": clean_wdx,
|
||||
"sentence_spans": sentence_spans
|
||||
}
|
||||
|
||||
|
||||
aligned_segments: List[SingleAlignedSegment] = []
|
||||
|
||||
|
||||
# 2. Get prediction matrix from alignment model & align
|
||||
for sdx, segment in enumerate(transcript):
|
||||
|
||||
|
||||
t1 = segment["start"]
|
||||
t2 = segment["end"]
|
||||
text = segment["text"]
|
||||
@@ -247,7 +248,7 @@ def align(
|
||||
)
|
||||
else:
|
||||
lengths = None
|
||||
|
||||
|
||||
with torch.inference_mode():
|
||||
if model_type == "torchaudio":
|
||||
emissions, _ = model(waveform_segment.to(device), lengths=lengths)
|
||||
@@ -304,7 +305,7 @@ def align(
|
||||
word_idx += 1
|
||||
elif cdx == len(text) - 1 or text[cdx+1] == " ":
|
||||
word_idx += 1
|
||||
|
||||
|
||||
char_segments_arr = pd.DataFrame(char_segments_arr)
|
||||
|
||||
aligned_subsegments = []
|
||||
@@ -333,7 +334,7 @@ def align(
|
||||
word_end = word_chars["end"].max()
|
||||
word_score = round(word_chars["score"].mean(), 3)
|
||||
|
||||
# -1 indicates unalignable
|
||||
# -1 indicates unalignable
|
||||
word_segment = {"word": word_text}
|
||||
|
||||
if not np.isnan(word_start):
|
||||
@@ -344,7 +345,7 @@ def align(
|
||||
word_segment["score"] = word_score
|
||||
|
||||
sentence_words.append(word_segment)
|
||||
|
||||
|
||||
aligned_subsegments.append({
|
||||
"text": sentence_text,
|
||||
"start": sentence_start,
|
||||
@@ -424,7 +425,7 @@ def get_wildcard_emission(frame_emission, tokens, blank_id):
|
||||
wildcard_mask = (tokens == -1)
|
||||
|
||||
# Get scores for non-wildcard positions
|
||||
regular_scores = frame_emission[tokens.clamp(min=0)] # clamp to avoid -1 index
|
||||
regular_scores = frame_emission[tokens.clamp(min=0).long()] # clamp to avoid -1 index
|
||||
|
||||
# Create a mask and compute the maximum value without modifying frame_emission
|
||||
max_valid_score = frame_emission.clone() # Create a copy
|
||||
|
||||
+5
-1
@@ -401,7 +401,11 @@ def load_model(
|
||||
if vad_method == "silero":
|
||||
vad_model = Silero(**default_vad_options)
|
||||
elif vad_method == "pyannote":
|
||||
vad_model = Pyannote(torch.device(device), use_auth_token=None, **default_vad_options)
|
||||
if device == 'cuda':
|
||||
device_vad = f'cuda:{device_index}'
|
||||
else:
|
||||
device_vad = device
|
||||
vad_model = Pyannote(torch.device(device_vad), use_auth_token=None, **default_vad_options)
|
||||
else:
|
||||
raise ValueError(f"Invalid vad_method: {vad_method}")
|
||||
|
||||
|
||||
@@ -52,23 +52,3 @@ class Vad:
|
||||
|
||||
return merged_segments
|
||||
|
||||
# Unused function
|
||||
@staticmethod
|
||||
def merge_vad(vad_arr, pad_onset=0.0, pad_offset=0.0, min_duration_off=0.0, min_duration_on=0.0):
|
||||
active = Annotation()
|
||||
for k, vad_t in enumerate(vad_arr):
|
||||
region = Segment(vad_t[0] - pad_onset, vad_t[1] + pad_offset)
|
||||
active[region, k] = 1
|
||||
|
||||
if pad_offset > 0.0 or pad_onset > 0.0 or min_duration_off > 0.0:
|
||||
active = active.support(collar=min_duration_off)
|
||||
|
||||
# remove tracks shorter than min_duration_on
|
||||
if min_duration_on > 0:
|
||||
for segment, track in list(active.itertracks()):
|
||||
if segment.duration < min_duration_on:
|
||||
del active[segment, track]
|
||||
|
||||
active = active.for_json()
|
||||
active_segs = pd.DataFrame([x['segment'] for x in active['content']])
|
||||
return active_segs
|
||||
Reference in new issue
Block a user