Compare commits

...
9 Commits
Author SHA1 Message Date
Max Bain 064f737998 fix: default compute_type to float32 on CPU to avoid float16 ValueError 2026-02-22 16:12:55 +00:00
Claude-Assistant d8a078eed4 feat: expose avg_logprob per segment from ctranslate2 beam search
Pass through the average log probability (transcription confidence score)
from ctranslate2 to the final segment output. The field is NotRequired
so existing code constructing segments without it remains valid.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-19 07:48:34 +01:00
carlito 4017efc273 Merge pull request #1356 from m-bain/revert-1355-batch_wrap
Revert "Batch wrap"
2026-02-17 13:34:20 +00:00
carlito 0e073d418c Revert "Batch wrap" 2026-02-17 13:33:06 +00:00
carlito 422c44f952 Merge pull request #1355 from 1carlito/batch_wrap
Batch wrap
2026-02-17 13:31:15 +00:00
carlos 1b6a3b78a6 [feat] First batch wrap around
[New File] benchmark testing
2026-02-17 13:30:03 +00:00
carlos de5fa652a1 [modification]
added and condition before streams, existing logic is not chnaged.
2026-02-16 15:43:01 +00:00
carlos e33bb1ec25 Although the existing commit worked, inital prompt was in the loop no need. 2026-02-16 15:43:01 +00:00
carlos 1430e43300 [fix] Batch context is updated each time.
It works with the initial prompt added.

Ran pdb to make sure and check output.

Long audio works.

Existing Logic is correct without flag.
2026-02-16 15:43:01 +00:00
4 changed files with 39 additions and 8 deletions

No files matched your search

+2 -2
View File
@@ -16,10 +16,10 @@ def cli():
parser.add_argument("--model", default="small", help="name of the Whisper model to use")
parser.add_argument("--model_cache_only", type=str2bool, default=False, help="If True, will not attempt to download models, instead using cached models from --model_dir")
parser.add_argument("--model_dir", type=str, default=None, help="the path to save model files; uses ~/.cache/whisper by default")
parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu", help="device to use for PyTorch inference")
parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu", help="device type to use for PyTorch inference (e.g. cpu, cuda)")
parser.add_argument("--device_index", default=0, type=int, help="device index to use for FasterWhisper inference")
parser.add_argument("--batch_size", default=8, type=int, help="the preferred batch size for inference")
parser.add_argument("--compute_type", default="float16", type=str, choices=["float16", "float32", "int8"], help="compute type for computation")
parser.add_argument("--compute_type", default="default", type=str, choices=["default", "float16", "float32", "int8"], help="compute type for computation; 'default' uses float16 on GPU, float32 on CPU")
parser.add_argument("--output_dir", "-o", type=str, default=".", help="directory to save the outputs")
parser.add_argument("--output_format", "-f", type=str, default="all", choices=["all", "srt", "vtt", "txt", "tsv", "json", "aud"], help="format of the output file; if not specified, all available formats will be produced")
+11 -2
View File
@@ -216,6 +216,7 @@ def align(
t1 = segment["start"]
t2 = segment["end"]
text = segment["text"]
avg_logprob = segment.get("avg_logprob")
aligned_seg: SingleAlignedSegment = {
"start": t1,
@@ -225,6 +226,9 @@ def align(
"chars": None,
}
if avg_logprob is not None:
aligned_seg["avg_logprob"] = avg_logprob
if return_char_alignments:
aligned_seg["chars"] = []
@@ -353,12 +357,15 @@ def align(
sentence_words.append(word_segment)
aligned_subsegments.append({
subsegment = {
"text": sentence_text,
"start": sentence_start,
"end": sentence_end,
"words": sentence_words,
})
}
if avg_logprob is not None:
subsegment["avg_logprob"] = avg_logprob
aligned_subsegments.append(subsegment)
if return_char_alignments:
curr_chars = curr_chars[["char", "start", "end", "score"]]
@@ -376,6 +383,8 @@ def align(
agg_dict["text"] = "".join
if return_char_alignments:
agg_dict["chars"] = "sum"
if avg_logprob is not None:
agg_dict["avg_logprob"] = "first"
aligned_subsegments= aligned_subsegments.groupby(["start", "end"], as_index=False).agg(agg_dict)
aligned_subsegments = aligned_subsegments.to_dict('records')
aligned_segments += aligned_subsegments
+19 -4
View File
@@ -70,10 +70,17 @@ class WhisperModel(faster_whisper.WhisperModel):
suppress_tokens=options.suppress_tokens,
no_repeat_ngram_size=options.no_repeat_ngram_size,
repetition_penalty=options.repetition_penalty,
return_scores=True,
)
tokens_batch = [x.sequences_ids[0] for x in result]
avg_logprobs = []
for res in result:
seq_len = len(res.sequences_ids[0])
cum_logprob = res.scores[0] * (seq_len ** options.length_penalty)
avg_logprobs.append(cum_logprob / (seq_len + 1))
def decode_batch(tokens: List[List[int]]) -> List[str]:
res = []
for tk in tokens:
@@ -83,7 +90,7 @@ class WhisperModel(faster_whisper.WhisperModel):
text = decode_batch(tokens_batch)
return text
return {'text': text, 'avg_logprob': avg_logprobs}
def encode(self, features: np.ndarray) -> ctranslate2.StorageView:
# When the model is running on multiple GPUs, the encoder output should be moved
@@ -161,7 +168,7 @@ class FasterWhisperPipeline(Pipeline):
def _forward(self, model_inputs):
outputs = self.model.generate_segment_batched(model_inputs['inputs'], self.tokenizer, self.options)
return {'text': outputs}
return outputs
def postprocess(self, model_outputs):
return model_outputs
@@ -262,15 +269,18 @@ class FasterWhisperPipeline(Pipeline):
percent_complete = base_progress / 2 if combined_progress else base_progress
print(f"Progress: {percent_complete:.2f}%...")
text = out['text']
avg_logprob = out['avg_logprob']
if batch_size in [0, 1, None]:
text = text[0]
avg_logprob = avg_logprob[0]
if verbose:
print(f"Transcript: [{round(vad_segments[idx]['start'], 3)} --> {round(vad_segments[idx]['end'], 3)}] {text}")
segments.append(
{
"text": text,
"start": round(vad_segments[idx]['start'], 3),
"end": round(vad_segments[idx]['end'], 3)
"end": round(vad_segments[idx]['end'], 3),
"avg_logprob": avg_logprob,
}
)
@@ -303,7 +313,7 @@ def load_model(
whisper_arch: str,
device: str,
device_index=0,
compute_type="float16",
compute_type="default",
asr_options: Optional[dict] = None,
language: Optional[str] = None,
vad_model: Optional[Vad]= None,
@@ -321,6 +331,7 @@ def load_model(
whisper_arch - The name of the Whisper model to load.
device - The device to load the model on.
compute_type - The compute type to use for the model.
Use "default" to automatically select based on device (float16 for GPU, float32 for CPU).
vad_model - The vad model to manually assign.
vad_method - The vad method to use. vad_model has a higher priority if it is not None.
options - A dictionary of options to use for the model.
@@ -333,6 +344,10 @@ def load_model(
A Whisper pipeline.
"""
if compute_type == "default":
compute_type = "float16" if device == "cuda" else "float32"
logger.info(f"Compute type not specified, defaulting to {compute_type} for device {device}")
if whisper_arch.endswith(".en"):
language = "en"
+7
View File
@@ -1,5 +1,10 @@
from typing import TypedDict, Optional, List, Tuple
try:
from typing import NotRequired
except ImportError:
from typing_extensions import NotRequired
class SingleWordSegment(TypedDict):
"""
@@ -28,6 +33,7 @@ class SingleSegment(TypedDict):
start: float
end: float
text: str
avg_logprob: NotRequired[float]
class SegmentData(TypedDict):
@@ -49,6 +55,7 @@ class SingleAlignedSegment(TypedDict):
start: float
end: float
text: str
avg_logprob: NotRequired[float]
words: List[SingleWordSegment]
chars: Optional[List[SingleCharSegment]]