Merge branch 'rakuri255:main' into main

This commit is contained in:
taynaron authored and GitHub committed 2026-04-03 11:49:41 -04:00
commit 78cb219336
46 files changed
+5159 -1287

No files matched your search

@@ -0,0 +1,88 @@
name: Build container image
on:
push:
workflow_dispatch:
env:
GITHUB_REPOSITORY_NAME_PART_SLUG_URL: '' # filled by rlespinasse/github-slug-action@v5
REGISTRY_USER: ${{ github.actor }}
REGISTRY_PASSWORD: ${{ github.token }}
jobs:
push-ghcr:
name: Build and push image
runs-on: ubuntu-22.04
permissions:
contents: read
packages: write
attestations: write
id-token: write
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Inject enhanced GitHub environment variables
uses: rlespinasse/github-slug-action@v5 # https://github.com/rlespinasse/github-slug-action
- name: lowercase IMAGE_REGISTRY
run: |
echo "IMAGE_REGISTRY=ghcr.io/${GITHUB_REPOSITORY_OWNER_PART_SLUG@L}" >> "${GITHUB_ENV}"
- name: Clear space
run: |
df -h /
sudo rm -rf \
"$AGENT_TOOLSDIRECTORY" \
/opt/google/chrome \
/opt/microsoft/msedge \
/opt/microsoft/powershell \
/opt/pipx \
/usr/lib/mono \
/usr/local/julia* \
/usr/local/lib/android \
/usr/local/lib/node_modules \
/usr/local/share/chromium \
/usr/local/share/powershell \
/usr/share/dotnet \
/usr/share/swift
df -h /
# Determine tags using Docker Metadata action https://github.com/docker/metadata-action
- name: Docker Metadata
id: docker-metadata
uses: docker/metadata-action@v5
with:
tags: |
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
type=edge
type=sha
type=ref,event=branch
type=ref,event=pr
type=schedule
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
# Build image using Buildah action https://github.com/redhat-actions/buildah-build
- name: Build Image
id: build_image
uses: redhat-actions/buildah-build@v2
with:
image: ${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG_URL }}
tags: ${{ steps.docker-metadata.outputs.tags }}
containerfiles: |
./Dockerfile
oci: true
layers: true
labels: |
${{ steps.docker-metadata.outputs.labels }}
# Push the image to GHCR (Image Registry) https://github.com/redhat-actions/push-to-registry
- name: Push To Registry
uses: redhat-actions/push-to-registry@v2
id: push
with:
image: ${{ steps.build_image.outputs.image }}
tags: ${{ steps.build_image.outputs.tags }}
registry: ${{ env.IMAGE_REGISTRY }}
username: ${{ env.REGISTRY_USER }}
password: ${{ env.REGISTRY_PASSWORD }}
extra-args: |
--disable-content-trust
- name: Echo outputs
run: |
echo "${{ toJSON(steps.push.outputs) }}"
-28
View File
@@ -1,28 +0,0 @@
name: docker
on:
push:
branches:
- 'main'
jobs:
docker:
environment: docker
runs-on: ubuntu-latest
steps:
- name: Delete huge unnecessary tools folder
run: rm -rf /opt/hostedtoolcache
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v6
with:
push: true
tags: rakuri255/ultrasinger:latest
+28 -13
View File
@@ -3,11 +3,13 @@ name: Check Requirements
on:
push:
paths:
- "requirements.txt"
- "pyproject.toml"
- "uv.lock"
pull_request:
paths:
- "requirements.txt"
- "pyproject.toml"
- "uv.lock"
jobs:
build:
@@ -21,22 +23,35 @@ jobs:
TARGET: macOS
- os: windows-latest
TARGET: Windows
python-version: [ "3.10" ]
python-version: [ "3.12" ]
steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v6
- name: Set up Python 3.10
uses: actions/setup-python@v4
- name: Set up Python 3.12
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
- name: Install uv
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Install gpu dependencies
if: matrix.TARGET != 'macOS'
run: pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu117 --index-url https://download.pytorch.org/whl/cu117
curl -LsSf https://astral.sh/uv/install.sh | sh
shell: bash
- name: Add uv to PATH (Unix)
if: matrix.TARGET != 'Windows'
run: echo "$HOME/.local/bin" >> $GITHUB_PATH
shell: bash
- name: Add uv to PATH (Windows)
if: matrix.TARGET == 'Windows'
run: echo "$env:USERPROFILE\.local\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
shell: pwsh
- name: Sync dependencies from pyproject.toml
run: uv sync
shell: bash
+2 -2
View File
@@ -14,10 +14,10 @@ jobs:
python-version: ["3.10"]
steps:
- name: Checkout code
uses: actions/checkout@v3
uses: actions/checkout@v6
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
+11 -8
View File
@@ -7,22 +7,25 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.10" ]
python-version: [ "3.12" ]
steps:
- name: Checkout code
uses: actions/checkout@v3
uses: actions/checkout@v6
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
- name: Install uv
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
#pip install -e .
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: Sync dependencies from pyproject.toml
run: uv sync
- name: Run pytest
run: python -m pytest --import-mode=append pytest/
run: uv run pytest --import-mode=append pytest/
+32 -15
View File
@@ -1,27 +1,44 @@
FROM nvidia/cuda:12.6.3-runtime-ubuntu22.04
FROM nvidia/cuda:12.8.1-runtime-ubuntu22.04
# note: the python3-pip package contains Python 3.10 on Ubuntu 22.04
# Set timezone and configure non-interactive installation
ENV DEBIAN_FRONTEND=noninteractive
ENV TZ=UTC
# Install Python 3.12 from deadsnakes PPA and build tools
RUN apt-get update \
&& apt-get install git python3-pip python3.10-venv ffmpeg curl -y \
&& apt-get install -y software-properties-common \
&& add-apt-repository ppa:deadsnakes/ppa -y \
&& apt-get update \
&& apt-get install -y git python3.12 python3.12-venv python3.12-dev ffmpeg curl tzdata \
build-essential gcc g++ make \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
COPY . /app/UltraSinger
# Install uv
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
ENV PATH="/root/.local/bin:$PATH"
ENV UV_LINK_MODE=copy
# copy pyproject.toml first to leverage container image build cache
COPY ./pyproject.toml /app/UltraSinger/pyproject.toml
# Need to copy some minimal source structure for editable install
RUN mkdir -p /app/UltraSinger/src
WORKDIR /app/UltraSinger
# Install dependencies from pyproject.toml directly without venv (container is already isolated)
# Using build isolation (without --no-build-isolation) so uv handles all build dependencies automatically
RUN uv pip install --system --python 3.12 -e .
# Install PyTorch with CUDA support (override the CPU version from pyproject.toml)
RUN uv pip install --system --python 3.12 torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu128 --reinstall
# copy sources late to allow for caching of layers which contain all the dependencies
COPY . /app/UltraSinger
# no need to run as root
RUN chown -R 1000:1000 /app/UltraSinger
USER 1000:1000
# setup venv
ENV VIRTUAL_ENV=/app/UltraSinger/.venv
RUN python3 -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
# install dependencies
RUN pip install --no-cache-dir -r requirements.txt \
&& pip install --no-cache-dir torch==2.3.0 torchvision==0.18.0 torchaudio==2.3.0 --index-url https://download.pytorch.org/whl/cu121 \
&& pip install --no-cache-dir tensorflow[and-cuda]==2.16.1
WORKDIR /app/UltraSinger/src
CMD ["bash" ]
CMD ["bash"]
+51 -62
View File
@@ -17,7 +17,7 @@
> ⚠️ _This project is still under development!_
UltraSinger is a tool to automatically create UltraStar.txt, midi and notes from music.
UltraSinger is a tool to automatically create UltraStar.txt, midi and notes from music.
It automatically pitches UltraStar files, adding text and tapping to UltraStar files and creates separate UltraStar karaoke files.
It also can re-pitch current UltraStar files and calculates the possible in-game score.
@@ -69,39 +69,39 @@ This will help me a lot to keep this project alive and improve it.
### Installation
* Install Python 3.10 **(older and newer versions has some breaking changes)**. [Download](https://www.python.org/downloads/)
* Install Python 3.12 **(older/newer versions may have compatibility issues)**. [Download](https://www.python.org/downloads/)
* Also download or install ffmpeg with PATH. [Download](https://www.ffmpeg.org/download.html)
* Go to folder `install` and run install script for your OS.
* Choose `GPU` if you have an nvidia CUDA GPU.
* Choose `CPU` if you don't have an nvidia CUDA GPU.
* Go to folder `install` and run install script for your OS:
* Choose `GPU` if you have an NVIDIA CUDA GPU.
* Choose `CPU` if you don't have an NVIDIA GPU or want CPU-only processing.
### Run
* In root folder just run `run_on_windows.bat` or `run_on_linux.sh` to start the app.
* Now you can use the UltraSinger source code with `py UltraSinger.py [opt] [mode] [transcription] [pitcher] [extra]`. See [How to use](#how-to-use) for more information.
* In root folder just run `run_on_windows.bat`, `run_on_linux.sh` or `run_on_mac.command` to start the app.
* Now you can use the UltraSinger source code with `py UltraSinger.py [opt] [mode] [transcription] [pitcher] [extra]`. See [How to use](#-how-to-use-the-app) for more information.
## 📖 How to use the App
_Not all options working now!_
```commandline
UltraSinger.py [opt] [mode] [transcription] [pitcher] [extra]
[opt]
-h This help text.
-i Ultrastar.txt
audio like .mp3, .wav, youtube link
audio/video like .mp3, .mp4, .wav, youtube link
-o Output folder
[mode]
## if INPUT is audio ##
default (Full Automatic Mode) - Creates all, depending on command line options
--interactive - Interactive Mode. All options are asked at runtime for easier configuration
# Single file creation selection is in progress, you currently getting all!
(-u Create ultrastar txt file) # In Progress
(-m Create midi file) # In Progress
(-s Create sheet file) # In Progress
## if INPUT is ultrastar.txt ##
default Creates all
@@ -117,13 +117,8 @@ _Not all options working now!_
--language Override the language detected by whisper, does not affect transcription but steps after transcription
--whisper_batch_size Reduce if low on GPU mem >> ((default) is 16)
--whisper_compute_type Change to "int8" if low on GPU mem (may reduce accuracy) >> ((default) is "float16" for cuda devices, "int8" for cpu)
--keep_numbers Numbers will be transcribed as numerics instead of as words
[pitcher]
# Default is crepe
--crepe tiny|full >> ((default) is full)
--crepe_step_size unit is miliseconds >> ((default) is 10)
--keep_numbers Numbers will be transcribed as numerics instead of as words
[extra]
--disable_hyphenation Disable word hyphenation. Hyphenation is enabled by default.
--disable_separation Disable track separation. Track separation is enabled by default.
@@ -131,18 +126,18 @@ _Not all options working now!_
--create_audio_chunks Enable creation of audio chunks. Audio chunks are disabled by default.
--keep_cache Keep cache folder after creation. Cache folder is removed by default.
--plot Enable creation of plots. Plots are disabled by default.
--quantize_to_key Quantize notes to detected musical key. Removes pitch slides and out-of-key notes. >> ((default) is enabled)
--format_version 0.3.0|1.0.0|1.1.0|1.2.0 >> ((default) is 1.2.0)
--musescore_path path to MuseScore executable
--keep_numbers Transcribe numbers as digits and not words
--ffmpeg Path to ffmpeg and ffprobe executable
[yt-dlp]
--cookiefile File name where cookies should be read from
[device]
--force_cpu Force all steps to be processed on CPU.
--force_whisper_cpu Only whisper will be forced to cpu
--force_crepe_cpu Only crepe will be forced to cpu
```
For standard use, you only need to use [opt]. All other options are optional.
@@ -155,7 +150,7 @@ default (Full Automatic Mode) - Creates all, depending on command line options
```commandline
--interactive
```
#### Audio (full automatic)
#### Audio / Video (full automatic)
##### Local file
@@ -196,8 +191,8 @@ For the first test run, use the `tiny`, to be accurate use the `large-v2` model.
##### Whisper languages
Currently provided default language models are `en, fr, de, es, it, ja, zh, nl, uk, pt`.
If the language is not in this list, you need to find a phoneme-based ASR model from
Currently provided default language models are `en, fr, de, es, it, ja, zh, nl, uk, pt`.
If the language is not in this list, you need to find a phoneme-based ASR model from
[🤗 huggingface model hub](https://huggingface.co). It will download automatically.
Example for romanian:
@@ -207,9 +202,9 @@ Example for romanian:
#### ✍️ Hyphenation
Is on by default. Can also be deactivated if hyphenation does not produce
anything useful. Note that the word is simply split,
without paying attention to whether the separated word really
Is on by default. Can also be deactivated if hyphenation does not produce
anything useful. Note that the word is simply split,
without paying attention to whether the separated word really
starts at the place or is heard. To disable:
```commandline
@@ -218,24 +213,28 @@ starts at the place or is heard. To disable:
### 👂 Pitcher
Pitching is done with the `crepe` model.
Also consider that a bigger model is more accurate, but also takes longer to pitch.
For just testing you should use `tiny`.
If you want solid accurate, then use the `full` model.
```commandline
-i XYZ --crepe full
```
Pitching is done with the `SwiftF0` model, which is faster and more accurate than CREPE.
SwiftF0 automatically detects pitch frequencies between 46.875 Hz (G1) and 2093.75 Hz (C7).
UltraSinger uses 60hz and 400hz
### 👄 Separation
The vocals are separated from the audio before they are passed to the models. If problems occur with this,
The vocals are separated from the audio before they are passed to the models. If problems occur with this,
you have the option to disable this function; in which case the original audio file is used instead.
```commandline
-i XYZ --disable_separation
-i XYZ --disable_separation
```
### 🎵 Key Quantization
By default, UltraSinger quantizes all detected notes to the detected musical key of the song. This helps to:
* Remove pitch slides and vocal transitions between notes
* Correct out-of-key notes from pitch detection errors
If you want to keep the raw pitch detection without quantization, set `quantize_to_key = False`.
For most songs quantization should produces better results.
### Sheet Music
For Sheet Music generation you need to have `MuseScore` installed on your system.
@@ -262,11 +261,11 @@ You can choose between different format versions. The default is `1.2.0`.
### 🏆 Ultrastar Score Calculation
The score that the singer in the audio would receive will be measured.
You get 2 scores, simple and accurate. You wonder where the difference is?
Ultrastar is not interested in pitch hights. As long as it is in the pitch range A-G you get one point.
This makes sense for the game, because otherwise men don't get points for high female voices and women don't get points
for low male voices. Accurate is the real tone specified in the txt. I had txt files where the pitch was in a range not
The score that the singer in the audio would receive will be measured.
You get 2 scores, simple and accurate. You wonder where the difference is?
Ultrastar is not interested in pitch hights. As long as it is in the pitch range A-G you get one point.
This makes sense for the game, because otherwise men don't get points for high female voices and women don't get points
for low male voices. Accurate is the real tone specified in the txt. I had txt files where the pitch was in a range not
singable by humans, but you could still reach the 10k points in the game. The accuracy is important here, because from
this MIDI and sheet are created. And you also want to have accurate files
@@ -275,34 +274,24 @@ this MIDI and sheet are created. And you also want to have accurate files
With a GPU you can speed up the process. Also the quality of the transcription and pitching is better.
You need a cuda device for this to work. Sorry, there is no cuda device for macOS.
You need an NVIDIA CUDA device for this to work. Sorry, there is no CUDA device for macOS.
It is optional (but recommended) to install the cuda driver for your gpu: see [driver](https://developer.nvidia.com/cuda-downloads).
Install torch with cuda separately in your `venv`. See [tourch+cuda](https://pytorch.org/get-started/locally/).
Also check you GPU cuda support. See [cuda support](https://gist.github.com/standaloneSA/99788f30466516dbcc00338b36ad5acf)
For GPU support on Windows and Linux, the installation script automatically installs PyTorch with CUDA support.
Command for `pip`:
It is optional (but recommended) to install the CUDA driver for your GPU: see [CUDA driver](https://developer.nvidia.com/cuda-downloads).
Also check your GPU CUDA support. See [CUDA support](https://gist.github.com/standaloneSA/99788f30466516dbcc00338b36ad5acf)
For manual installation, you can use:
```bash
uv pip install --index-url https://download.pytorch.org/whl/cu121 torch torchvision torchaudio
```
pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu117 --index-url https://download.pytorch.org/whl/cu117
```
When you want to use `conda` instead you need a [different installation command](https://pytorch.org/get-started/locally/).
#### Considerations for Windows users
The pitch tracker used by UltraSinger (crepe) uses TensorFlow as its backend.
TensorFlow dropped GPU support for Windows for versions >2.10 as you can see in this [release note](https://github.com/tensorflow/tensorflow/releases/tag/v2.11.1) and their [installation instructions](https://www.tensorflow.org/install/pip#windows-native).
For now UltraSinger runs the latest version available that still supports GPUs on windows.
For running later versions of TensorFlow on windows while still taking advantage of GPU support the suggested solution is to [run UltraSinger in a container](container/README.md).
#### Crashes due to low VRAM
If something crashes because of low VRAM then use a smaller model.
If something crashes because of low VRAM then use a smaller Whisper model.
Whisper needs more than 8GB VRAM in the `large` model!
You can also force cpu usage with the extra option `--force_cpu`.
You can also force CPU usage with the extra option `--force_cpu`.
### 📦 Containerized (Docker or Podman)
+9 -2
View File
@@ -1,5 +1,5 @@
# Version: 0.0.13dev6
Date: 2024.12.26
# Version: 0.0.13dev16
Date: 2026.03.04
- Changes:
- Download Cover from MusicBrainz
- Fix selection of whisper model
@@ -10,6 +10,13 @@ Date: 2024.12.26
- Use user defined ffmpeg path
- Show GPU Name and VRAM
- No linebreak before the last 'E' word for UltraStar txt files
- Download yt video/audio only once
- Support for video as input
- Optimise scale detection
- Added quantization by key
- Changed installer to uv
- Drop crepe for SwiftF0
- upgrade to python 3.12
# Version: 0.0.12
Date: 2024.12.19
+62 -46
View File
@@ -1,48 +1,64 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"gpuType": "T4"
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
},
"accelerator": "GPU"
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"gpuType": "T4"
},
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Hki_kX_kSbZX"
},
"outputs": [],
"source": [
"%cd /content\n",
"!git clone https://github.com/rakuri255/UltraSinger.git\n",
"%cd /content/UltraSinger\n",
"!pip install --no-cache-dir -r requirements.txt\n",
"!pip install --no-cache-dir torch==2.3.0 torchvision==0.18.0 torchaudio==2.3.0 --index-url https://download.pytorch.org/whl/cu121\n",
"!pip install --no-cache-dir tensorflow[and-cuda]==2.17.1\n",
"%cd /content/UltraSinger/src"
]
},
{
"cell_type": "code",
"source": [
"!python UltraSinger.py -i https://www.youtube.com/watch?v=YwNs1Z0qRY0 -o /content/output"
],
"metadata": {
"id": "O0j4vUW0YAG2"
},
"execution_count": null,
"outputs": []
}
]
}
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Hki_kX_kSbZX"
},
"outputs": [],
"source": [
"%cd /content\n",
"\n",
"# Remove existing directory if present\n",
"#!rm -rf UltraSinger\n",
"\n",
"# Clone specific branch\n",
"!git clone https://github.com/rakuri255/UltraSinger.git\n",
"%cd /content/UltraSinger\n",
"\n",
"# Install uv\n",
"!curl -LsSf https://astral.sh/uv/install.sh | sh\n",
"\n",
"# Add uv to PATH and configure matplotlib\n",
"import os\n",
"os.environ['PATH'] = f\"/root/.local/bin:{os.environ['PATH']}\"\n",
"os.environ['UV_LINK_MODE'] = 'copy'\n",
"os.environ['MPLBACKEND'] = 'Agg'\n",
"\n",
"# Sync dependencies from pyproject.toml\n",
"!uv sync\n",
"\n",
"# Install PyTorch with CUDA support\n",
"!uv pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu128 --force-reinstall\n",
"\n",
"%cd /content/UltraSinger/src"
]
},
{
"cell_type": "code",
"source": "!../.venv/bin/python UltraSinger.py -i https://www.youtube.com/watch?v=YwNs1Z0qRY0 -o /content/output\n",
"metadata": {
"id": "O0j4vUW0YAG2"
},
"execution_count": null,
"outputs": []
}
]
}
+6 -4
View File
@@ -7,9 +7,11 @@
```commandline
git clone https://github.com/rakuri255/UltraSinger.git
```
1. build the container image
```commandline
1. build or pull the container image
```bash
podman build -t ultrasinger .
# or
podman pull ghcr.io/rakuri255/ultrasinger:<version> # replace <version> with the desired version or 'latest'
```
1. run the container (note that the first time you run the container models will be downloaded which may take a while)
1. PowerShell:
@@ -20,7 +22,7 @@
--device nvidia.com/gpu=all `
-v $env:USERPROFILE\.cache:/app/UltraSinger/src/.cache `
-v <desired-output-folder>:/app/UltraSinger/src/output `
ultrasinger `
ghcr.io/rakuri255/ultrasinger `
python3 UltraSinger.py -i <refer to top-level README.md>
# explanation:
@@ -29,7 +31,7 @@
--device nvidia.com/gpu=all ` # optional, enables GPU acceleration if available, requires step 1.ii
-v $env:USERPROFILE\.cache:/app/UltraSinger/src/.cache ` # cache directory for models
-v <desired-output-folder>:/app/UltraSinger/src/output ` # output directory
ultrasinger ` # container image name, we built this in step 3
ghcr.io/rakuri255/ultrasinger ` # latest container image, set to 'ultrasinger' to use the locally built image
python3 UltraSinger.py -i <refer to top-level README.md> # run UltraSinger, refer to top-level README.md for all options
```
1. Bash:
+34 -6
View File
@@ -1,7 +1,35 @@
#!/bin/bash
cd ..
cd ..
python3.10 -m venv .venv
source .venv/bin/activate
pip install -r requirements.txt
pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1
set -e
cd "$(dirname "$0")"
cd ../..
# Set link mode to copy to avoid hardlink warnings
export UV_LINK_MODE=copy
# Install uv if not already installed
if ! command -v uv &> /dev/null; then
echo "Installing uv..."
curl -LsSf https://astral.sh/uv/install.sh | sh
# Update PATH for current session
export PATH="$HOME/.local/bin:$PATH"
fi
# Verify uv is available
if ! command -v uv &> /dev/null; then
echo "Error: uv could not be found or installed"
echo "Please ensure your shell PATH includes ~/.local/bin"
exit 1
fi
echo "uv version:"
uv --version
echo "Syncing dependencies with uv..."
uv sync --extra linux
echo "Installation completed successfully!"
echo "To run UltraSinger:"
echo " source .venv/bin/activate"
echo " cd src"
echo " py UltraSinger.py"
+35
View File
@@ -0,0 +1,35 @@
#!/bin/bash
set -e
cd "$(dirname "$0")"
cd ../..
# Set link mode to copy to avoid hardlink warnings
export UV_LINK_MODE=copy
# Install uv if not already installed
if ! command -v uv &> /dev/null; then
echo "Installing uv..."
curl -LsSf https://astral.sh/uv/install.sh | sh
# Update PATH for current session
export PATH="$HOME/.local/bin:$PATH"
fi
# Verify uv is available
if ! command -v uv &> /dev/null; then
echo "Error: uv could not be found or installed"
echo "Please ensure your shell PATH includes ~/.local/bin"
exit 1
fi
echo "uv version:"
uv --version
echo "Syncing dependencies with uv..."
uv sync --extra macos
echo "Installation completed successfully!"
echo "To run UltraSinger:"
echo " source .venv/bin/activate"
echo " cd src"
echo " py UltraSinger.py"
+96 -8
View File
@@ -1,9 +1,97 @@
@echo off
setlocal
cd ..
cd ..
py -3.10 -m venv .venv
SET VenvPythonPath=%CD%\.venv\Scripts\python.exe
call %VenvPythonPath% -m pip install -r requirements.txt
call %VenvPythonPath% -m pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1
endlocal
setlocal enabledelayedexpansion
:: Set link mode to copy to avoid hardlink warnings on different filesystems
set UV_LINK_MODE=copy
:: Navigate to project root
pushd "%~dp0"
cd /d ..\..
echo Current directory: %cd%
:: Update PATH to include uv installation directory
set "PATH=%USERPROFILE%\.local\bin;!PATH!"
:: Remove old virtual environment if it exists to force recreation with correct Python version
::if exist .venv (
:: echo Removing old virtual environment...
:: rmdir /s /q .venv
::)
:: First, find Python using to get full path
set "PYTHON_EXE="
for %%V in (3.12) do (
py -%%V --version >nul 2>&1
if !errorlevel! equ 0 (
:: Get the full path to the Python executable
for /f "delims=" %%P in ('py -%%V -c "import sys; print(sys.executable)"') do (
set "PYTHON_EXE=%%P"
)
goto :found_python
)
)
:: Fallback to direct Python installations
for %%P in (python3.12 python3 python) do (
where %%P >nul 2>&1
if !errorlevel! equ 0 (
set "PYTHON_EXE=%%P"
goto :found_python
)
)
:found_python
if "!PYTHON_EXE!"=="" (
echo Error: No Python 3.12 installation found
echo Please install Python 3.12 from python.org
echo Note: Python 3.13 is not yet supported due to dependency constraints
pause
exit /b 1
)
echo Using Python: !PYTHON_EXE!
"!PYTHON_EXE!" --version
:: Install uv if not already installed
where uv >nul 2>&1
if !errorlevel! neq 0 (
echo Installing uv...
powershell -NoProfile -Command "irm https://astral.sh/uv/install.ps1 | iex"
)
:: Wait a moment for uv to be available
timeout /t 2 /nobreak >nul
:: Verify uv is available
where uv >nul 2>&1
if !errorlevel! neq 0 (
echo Error: uv could not be found or installed
pause
exit /b 1
)
echo uv is ready
uv --version
echo Syncing dependencies with uv...
uv sync --extra windows --python "!PYTHON_EXE!"
if !errorlevel! neq 0 (
echo Error during uv sync
pause
exit /b 1
)
echo Installing PyTorch CPU version...
:: First remove any existing torch installation to avoid RECORD file issues
uv pip uninstall torch torchvision torchaudio -y 2>nul
:: Install PyTorch CPU version
uv pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision torchaudio
if !errorlevel! neq 0 (
echo Error during PyTorch installation
pause
exit /b 1
)
echo Installation completed successfully!
pause
+39 -6
View File
@@ -1,7 +1,40 @@
#!/bin/bash
cd ..
cd ..
python3.10 -m venv .venv
source .venv/bin/activate
pip install -r requirements.txt
pip install torch==2.3.1+cu118 torchvision==0.18.1+cu118 torchaudio==2.3.1+cu118 --index-url https://download.pytorch.org/whl/cu118
set -e
cd "$(dirname "$0")"
cd ../..
# Set link mode to copy to avoid hardlink warnings on different filesystems
export UV_LINK_MODE=copy
# Install uv if not already installed
if ! command -v uv &> /dev/null; then
echo "Installing uv..."
curl -LsSf https://astral.sh/uv/install.sh | sh
# Update PATH for current session
export PATH="$HOME/.local/bin:$PATH"
fi
# Verify uv is available
if ! command -v uv &> /dev/null; then
echo "Error: uv could not be found or installed"
echo "Please ensure your shell PATH includes ~/.local/bin"
exit 1
fi
echo "uv version:"
uv --version
# Sync dependencies from pyproject.toml
echo "Syncing dependencies from pyproject.toml..."
uv sync --extra linux
# Install PyTorch with CUDA support
echo "Installing PyTorch with CUDA support..."
uv pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu128 --force-reinstall
echo "Installation completed successfully!"
echo "To run UltraSinger:"
echo " source .venv/bin/activate"
echo " cd src"
echo " python UltraSinger.py"
+97 -8
View File
@@ -1,9 +1,98 @@
@echo off
setlocal
cd ..
cd ..
py -3.10 -m venv .venv
SET VenvPythonPath=%CD%\.venv\Scripts\python.exe
call %VenvPythonPath% -m pip install -r requirements.txt
call %VenvPythonPath% -m pip install torch==2.3.1+cu118 torchvision==0.18.1+cu118 torchaudio==2.3.1+cu118 --index-url https://download.pytorch.org/whl/cu118
endlocal
setlocal enabledelayedexpansion
:: Set link mode to copy to avoid hardlink warnings on different filesystems
set UV_LINK_MODE=copy
:: Navigate to project root
pushd "%~dp0"
cd /d ..\..
echo Current directory: %cd%
:: Update PATH to include uv installation directory
set "PATH=%USERPROFILE%\.local\bin;!PATH!"
:: Remove old virtual environment if it exists to force recreation with correct Python version
::if exist .venv (
:: echo Removing old virtual environment...
:: rmdir /s /q .venv
::)
:: First, find Python using to get full path
set "PYTHON_EXE="
for %%V in (3.12) do (
py -%%V --version >nul 2>&1
if !errorlevel! equ 0 (
:: Get the full path to the Python executable
for /f "delims=" %%P in ('py -%%V -c "import sys; print(sys.executable)"') do (
set "PYTHON_EXE=%%P"
)
goto :found_python
)
)
:: Fallback to direct Python installations
for %%P in (python3.12 python3 python) do (
where %%P >nul 2>&1
if !errorlevel! equ 0 (
set "PYTHON_EXE=%%P"
goto :found_python
)
)
:found_python
if "!PYTHON_EXE!"=="" (
echo Error: No Python 3.12 installation found
echo Please install Python 3.12 from python.org
echo Note: Python 3.13 is not yet supported due to dependency constraints
pause
exit /b 1
)
echo Using Python: !PYTHON_EXE!
"!PYTHON_EXE!" --version
:: Install uv if not already installed
where uv >nul 2>&1
if !errorlevel! neq 0 (
echo Installing uv...
powershell -NoProfile -Command "irm https://astral.sh/uv/install.ps1 | iex"
)
:: Wait a moment for uv to be available
timeout /t 2 /nobreak >nul
:: Verify uv is available
where uv >nul 2>&1
if !errorlevel! neq 0 (
echo Error: uv could not be found or installed
pause
exit /b 1
)
echo uv is ready
uv --version
echo Syncing dependencies with uv...
uv sync --extra windows --python "!PYTHON_EXE!"
if !errorlevel! neq 0 (
echo Error during uv sync
pause
exit /b 1
)
echo Installing PyTorch with CUDA support (as recommended by WhisperX)...
:: First remove any existing torch installation to avoid RECORD file issues
uv pip uninstall torch torchvision torchaudio -y 2>nul
:: Install PyTorch 2.8.0 with CUDA for WhisperX compatibility
:: This version includes cuDNN 9.x which is compatible with the latest PyTorch
uv pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu128 --force-reinstall
if !errorlevel! neq 0 (
echo Error during PyTorch installation
pause
exit /b 1
)
echo Installation completed successfully!
pause
+100
View File
@@ -0,0 +1,100 @@
@echo off
echo ========================================
echo CUDA 11.x Libraries Fix for WhisperX/Pyannote
echo ========================================
echo.
echo This script will download and install CUDA 11.x DLLs
echo that are required by pyannote.audio (used by WhisperX)
echo.
echo PyTorch 2.8.0 includes CUDA 12.x, but pyannote.audio
echo still requires CUDA 11.x DLLs to function properly.
echo.
cd /d "%~dp0\.."
echo Checking for virtual environment...
if not exist ".venv\Scripts\activate.bat" (
echo Error: Virtual environment not found at .venv
echo Please create the virtual environment first.
pause
exit /b 1
)
echo Activating virtual environment...
call .venv\Scripts\activate.bat
if %errorlevel% neq 0 (
echo Error activating virtual environment
pause
exit /b 1
)
echo Creating temporary directory...
if not exist temp_cuda11 mkdir temp_cuda11
cd temp_cuda11
echo.
echo Downloading CUDA 11.8 Toolkit Libraries (Windows)...
echo This may take a few minutes...
echo.
echo Downloading cuBLAS 11.x...
powershell -Command "& {[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; Invoke-WebRequest -Uri 'https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-11.11.3.6-archive.zip' -OutFile 'cublas.zip'}"
if %errorlevel% neq 0 (
echo Error downloading cuBLAS 11.x
echo.
echo Please check your internet connection or download manually from:
echo https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/
echo.
cd ..
deactivate
pause
exit /b 1
)
echo.
echo Extracting CUDA 11.x Libraries...
if exist cublas.zip (
powershell -Command "Expand-Archive -Path 'cublas.zip' -DestinationPath '.' -Force"
)
echo.
echo Copying CUDA 11.x DLLs to PyTorch installation...
set "COPIED=0"
if exist "libcublas-windows-x86_64-11.11.3.6-archive\bin\*.dll" (
copy /Y "libcublas-windows-x86_64-11.11.3.6-archive\bin\*.dll" "%VIRTUAL_ENV%\Lib\site-packages\torch\lib\"
set "COPIED=1"
)
if "%COPIED%"=="0" (
echo Error: Could not find CUDA 11.x DLLs in extracted files
echo Expected path: libcublas-windows-x86_64-11.11.3.6-archive\bin\
cd ..
deactivate
pause
exit /b 1
)
echo.
echo Successfully copied CUDA 11.x DLLs!
echo Listing copied cuBLAS files:
dir "%VIRTUAL_ENV%\Lib\site-packages\torch\lib\cublas*.dll" /b 2>nul
echo.
echo Cleaning up temporary files...
cd ..
rmdir /s /q temp_cuda11
echo Deactivating virtual environment...
deactivate
echo.
echo ========================================
echo CUDA 11.x Fix completed successfully!
echo ========================================
echo.
echo You can now run UltraSinger without CUDA 11.x DLL errors.
echo.
pause
+102
View File
@@ -0,0 +1,102 @@
@echo off
echo ========================================
echo cuDNN 8.x Fix for WhisperX/Pyannote
echo ========================================
echo.
echo This script will download and install cuDNN 8.x DLLs
echo that are required by pyannote.audio (used by WhisperX)
echo.
echo PyTorch 2.8.0 includes cuDNN 9.x, but pyannote.audio
echo still requires cuDNN 8.x DLLs to function properly.
echo.
cd /d "%~dp0\.."
echo Checking for virtual environment...
if not exist ".venv\Scripts\activate.bat" (
echo Error: Virtual environment not found at .venv
echo Please create the virtual environment first.
pause
exit /b 1
)
echo Activating virtual environment...
call .venv\Scripts\activate.bat
if %errorlevel% neq 0 (
echo Error activating virtual environment
pause
exit /b 1
)
echo Creating temporary directory...
if not exist temp_cudnn mkdir temp_cudnn
cd temp_cudnn
echo.
echo Downloading cuDNN 8.9.7 for CUDA 11.x (Windows)...
echo This may take a few minutes...
powershell -Command "& {[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; Invoke-WebRequest -Uri 'https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/windows-x86_64/cudnn-windows-x86_64-8.9.7.29_cuda11-archive.zip' -OutFile 'cudnn.zip'}"
if %errorlevel% neq 0 (
echo Error downloading cuDNN 8.9.7
echo.
echo Alternative: Trying to extract from installed torch packages...
echo Searching for existing PyTorch installations with cuDNN 8.x...
REM Try to find cuDNN 8 DLLs in pip cache or other Python installations
powershell -Command "& {$found = $false; Get-ChildItem -Path $env:LOCALAPPDATA\pip\cache -Recurse -Filter 'cudnn64_8.dll' -ErrorAction SilentlyContinue | ForEach-Object {echo $_.FullName; $found = $true}; if (-not $found) {exit 1}}"
if %errorlevel% neq 0 (
echo Error: Could not find or download cuDNN 8.x DLLs
echo.
echo Please manually download cuDNN 8.9.7 from NVIDIA:
echo https://developer.nvidia.com/rdp/cudnn-archive
echo.
cd ..
deactivate
pause
exit /b 1
)
)
echo.
echo Extracting cuDNN 8.x DLLs...
if exist cudnn.zip (
powershell -Command "Expand-Archive -Path 'cudnn.zip' -DestinationPath '.' -Force"
)
echo.
echo Copying cuDNN 8.x DLLs to PyTorch installation...
if exist "cudnn-windows-x86_64-8.9.7.29_cuda11-archive\bin\cudnn*.dll" (
copy /Y "cudnn-windows-x86_64-8.9.7.29_cuda11-archive\bin\cudnn*.dll" "%VIRTUAL_ENV%\Lib\site-packages\torch\lib\"
echo.
echo Successfully copied cuDNN 8.x DLLs!
echo Listing copied files:
dir "%VIRTUAL_ENV%\Lib\site-packages\torch\lib\cudnn*.dll" /b
) else (
echo Error: Could not find cuDNN DLLs in extracted files
echo Expected path: cudnn-windows-x86_64-8.9.7.29_cuda11-archive\bin\
cd ..
deactivate
pause
exit /b 1
)
echo.
echo Cleaning up temporary files...
cd ..
rmdir /s /q temp_cudnn
echo Deactivating virtual environment...
deactivate
echo.
echo ========================================
echo cuDNN 8.x Fix completed successfully!
echo ========================================
echo.
echo You can now run UltraSinger without cuDNN errors.
echo.
pause
+47 -10
View File
@@ -1,22 +1,59 @@
[build-system]
requires = ["setuptools", "setuptools-scm"]
build-backend = "setuptools.build_meta"
requires = ["hatchling", "setuptools", "wheel", "setuptools_scm"]
build-backend = "hatchling.build"
[project]
requires-python = "3.10"
name = "UltraSinger"
dynamic = ["dependencies", "optional-dependencies"]
[tool.setuptools.dynamic]
dependencies = { file = ["requirements.in"] }
optional-dependencies.test = { file = ["requirements-test.txt"] }
version = "0.0.13.dev16"
description = "A tool to create UltraStar karaoke files from audio files"
requires-python = ">=3.12,<3.14"
dependencies = [
"setuptools",
"scipy",
"whisperx",
"num2words",
"inputimeout",
"langcodes",
"language_data",
"packaging",
"librosa",
"swift-f0",
"pydub",
"demucs",
"ffmpeg-python",
"matplotlib",
"musicbrainzngs",
"python-Levenshtein",
"pretty-midi",
"unidecode",
"pyhyphen",
"tqdm",
"yt-dlp[default]",
"deno",
"music21",
"dataclasses",
"dataclasses-json",
"torch==2.8.0",
"torchaudio==2.8.0",
"torchvision==0.23.0",
]
[project.optional-dependencies]
dev = ["pytest"]
windows = []
linux = []
macos = []
dev = ["pytest", "isort", "black", "pylint"]
[dependency-groups]
dev = ["pytest", "isort", "black", "pylint"]
[tool.hatch.build.targets.wheel]
packages = ["src"]
[tool.isort]
profile = "black"
[tool.pytest.ini_options]
pythonpath = [
"src",
]
]
+1 -1
View File
@@ -18,7 +18,7 @@ class PitcherTest(unittest.TestCase):
test_output = root_dir + "/test_output"
# Act
pitched_data = test_subject.get_pitch_with_crepe_file(test_file_abs_path, 'full', device="cuda")
pitched_data = test_subject.get_pitch_with_file(test_file_abs_path)
# test_subject.get_pitch_with_crepe_file(test_file_abs_path, 'full', 'cpu', batch_size=1024)
plot(pitched_data, test_output, title="pitching test")
print("done")
@@ -29,7 +29,8 @@ class TestUltraStarParser(unittest.TestCase):
self.assertEqual(result, ("Test Artist - Test Title",
os.path.join("path", "to", "output", "Test Artist - Test Title"),
os.path.join("path", "to", "input", "test.mp3"),
mock_parse.return_value))
mock_parse.return_value,
"mp3"))
#
mock_parse.assert_called_once()
mock_dirname.assert_called_once()
@@ -104,26 +104,26 @@ class TestCreateUltrastarTxt(unittest.TestCase):
def default_values(default_ultrastar_class, ver):
expected_calls = []
if version.parse(ver) >= version.parse("1.0.0"):
expected_calls.append(f"#{UltrastarTxtTag.VERSION}:{default_ultrastar_class.version}\n")
expected_calls.append(f"#{UltrastarTxtTag.VERSION.value}:{default_ultrastar_class.version}\n")
expected_calls += [
f"#{UltrastarTxtTag.ARTIST}:{default_ultrastar_class.artist}\n",
f"#{UltrastarTxtTag.TITLE}:{default_ultrastar_class.title}\n",
f"#{UltrastarTxtTag.MP3}:{default_ultrastar_class.mp3}\n"
f"#{UltrastarTxtTag.ARTIST.value}:{default_ultrastar_class.artist}\n",
f"#{UltrastarTxtTag.TITLE.value}:{default_ultrastar_class.title}\n",
f"#{UltrastarTxtTag.MP3.value}:{default_ultrastar_class.mp3}\n"
]
if version.parse(ver) >= version.parse("1.1.0"):
expected_calls += [f"#{UltrastarTxtTag.AUDIO}:{default_ultrastar_class.audio}\n"]
expected_calls += [f"#{UltrastarTxtTag.AUDIO.value}:{default_ultrastar_class.audio}\n"]
if default_ultrastar_class.video is not None:
expected_calls += [
f"#{UltrastarTxtTag.VIDEO}:{default_ultrastar_class.video}\n"
f"#{UltrastarTxtTag.VIDEO.value}:{default_ultrastar_class.video}\n"
]
if version.parse(ver) >= version.parse("1.2.0"):
if default_ultrastar_class.videoUrl is not None:
expected_calls += [f"#{UltrastarTxtTag.VIDEOURL}:{default_ultrastar_class.videoUrl}\n"]
expected_calls += [f"#{UltrastarTxtTag.VIDEOURL.value}:{default_ultrastar_class.videoUrl}\n"]
expected_calls += [
f"#{UltrastarTxtTag.BPM}:390.0\n",
f"#{UltrastarTxtTag.GAP}:500\n",
f"#{UltrastarTxtTag.CREATOR}:{default_ultrastar_class.creator}\n",
f"#{UltrastarTxtTag.COMMENT}:{default_ultrastar_class.comment}\n",
f"#{UltrastarTxtTag.BPM.value}:390.0\n",
f"#{UltrastarTxtTag.GAP.value}:500\n",
f"#{UltrastarTxtTag.CREATOR.value}:{default_ultrastar_class.creator}\n",
f"#{UltrastarTxtTag.COMMENT.value}:{default_ultrastar_class.comment}\n",
": 0 52 1 UltraSinger \n",
": 65 39 2 is \n",
": 130 52 3 cool! \n",
@@ -136,26 +136,27 @@ class TestCreateUltrastarTxt(unittest.TestCase):
def full_values(default_ultrastar_class, ver):
expected_calls = []
if version.parse(ver) >= version.parse("1.0.0"):
expected_calls.append(f"#{UltrastarTxtTag.VERSION}:{default_ultrastar_class.version}\n")
expected_calls.append(f"#{UltrastarTxtTag.ARTIST}:{default_ultrastar_class.artist}\n")
expected_calls.append(f"#{UltrastarTxtTag.TITLE}:{default_ultrastar_class.title}\n")
expected_calls.append(f"#{UltrastarTxtTag.YEAR}:{default_ultrastar_class.year}\n")
expected_calls.append(f"#{UltrastarTxtTag.LANGUAGE}:German\n")
expected_calls.append(f"#{UltrastarTxtTag.GENRE}:{default_ultrastar_class.genre}\n")
expected_calls.append(f"#{UltrastarTxtTag.COVER}:{default_ultrastar_class.cover}\n")
expected_calls.append(f"#{UltrastarTxtTag.MP3}:{default_ultrastar_class.mp3}\n")
expected_calls.append(f"#{UltrastarTxtTag.VERSION.value}:{default_ultrastar_class.version}\n")
expected_calls.append(f"#{UltrastarTxtTag.ARTIST.value}:{default_ultrastar_class.artist}\n")
expected_calls.append(f"#{UltrastarTxtTag.TITLE.value}:{default_ultrastar_class.title}\n")
expected_calls.append(f"#{UltrastarTxtTag.YEAR.value}:{default_ultrastar_class.year}\n")
expected_calls.append(f"#{UltrastarTxtTag.LANGUAGE.value}:German\n")
expected_calls.append(f"#{UltrastarTxtTag.GENRE.value}:{default_ultrastar_class.genre}\n")
expected_calls.append(f"#{UltrastarTxtTag.COVER.value}:{default_ultrastar_class.cover}\n")
expected_calls.append(f"#{UltrastarTxtTag.MP3.value}:{default_ultrastar_class.mp3}\n")
if version.parse(ver) >= version.parse("1.1.0"):
expected_calls.append(f"#{UltrastarTxtTag.AUDIO}:{default_ultrastar_class.audio}\n")
expected_calls.append(f"#{UltrastarTxtTag.VOCALS}:{default_ultrastar_class.vocals}\n")
expected_calls.append(f"#{UltrastarTxtTag.INSTRUMENTAL}:{default_ultrastar_class.instrumental}\n")
expected_calls.append(f"#{UltrastarTxtTag.TAGS}:{default_ultrastar_class.tags}\n")
expected_calls.append(f"#{UltrastarTxtTag.VIDEO}:{default_ultrastar_class.video}\n")
expected_calls.append(f"#{UltrastarTxtTag.AUDIO.value}:{default_ultrastar_class.audio}\n")
expected_calls.append(f"#{UltrastarTxtTag.VOCALS.value}:{default_ultrastar_class.vocals}\n")
expected_calls.append(f"#{UltrastarTxtTag.INSTRUMENTAL.value}:{default_ultrastar_class.instrumental}\n")
expected_calls.append(f"#{UltrastarTxtTag.VIDEO.value}:{default_ultrastar_class.video}\n")
if version.parse(ver) >= version.parse("1.2.0"):
expected_calls.append(f"#{UltrastarTxtTag.VIDEOURL}:{default_ultrastar_class.videoUrl}\n")
expected_calls.append(f"#{UltrastarTxtTag.BPM}:390.0\n")
expected_calls.append(f"#{UltrastarTxtTag.GAP}:500\n")
expected_calls.append(f"#{UltrastarTxtTag.CREATOR}:{default_ultrastar_class.creator}\n")
expected_calls.append(f"#{UltrastarTxtTag.COMMENT}:{default_ultrastar_class.comment}\n")
expected_calls.append(f"#{UltrastarTxtTag.VIDEOURL.value}:{default_ultrastar_class.videoUrl}\n")
expected_calls.append(f"#{UltrastarTxtTag.BPM.value}:390.0\n")
expected_calls.append(f"#{UltrastarTxtTag.GAP.value}:500\n")
if version.parse(ver) >= version.parse("1.1.0"):
expected_calls.append(f"#{UltrastarTxtTag.TAGS.value}:{default_ultrastar_class.tags}\n")
expected_calls.append(f"#{UltrastarTxtTag.CREATOR.value}:{default_ultrastar_class.creator}\n")
expected_calls.append(f"#{UltrastarTxtTag.COMMENT.value}:{default_ultrastar_class.comment}\n")
expected_calls.append(": 0 52 1 UltraSinger \n")
expected_calls.append(": 65 39 2 is \n")
expected_calls.append(": 130 52 3 cool! \n")
-75
View File
@@ -1,75 +0,0 @@
# requirements.in
# Pitching
crepe
# machine learning platform (used for crepe)
tensorflow==2.10
# Audio to wavefile (used for crepe)
scipy
# Transcription
#whisperx
git+https://github.com/m-bain/whisperx.git
# Convert numbers to words in multilanguage (Used because of whisper)
num2words
# User imput timeout (used in whisper)
inputimeout
# langcodes for creates the language name from the language code. Used for Ultrastar txt
langcodes
# Version handling (Used in UltraStar txt)
packaging
# Audio analysis
librosa~=0.9.2
# Audio manipulation
pydub
# Audio Separation
demucs
# Ffmpeg wrapper
ffmpeg_python
# Plot engine
matplotlib
# For getting additional song info like cover, genre, year
musicbrainzngs
# For distance calculation (used in musicbrainz_client)
python_Levenshtein
# For midi file handling
pretty_midi
# Convert Unicode to ascii (used for midi)
unidecode
# For text hyphenation
#PyHyphen
PyHyphen-Reloaded
# Progress bar
tqdm
# yt
yt_dlp
# external cleanup tools
## Sort imports
isort
## Code formatter
black
## Code analyser
pylint
# For Tests
pytest
# Musical Analysis and Computational Musicology (Used for sheet)
music21
# Data structure
dataclasses
dataclasses_json
# Array support
numpy<2
-655
View File
@@ -1,655 +0,0 @@
#
# This file is autogenerated by pip-compile with Python 3.10
# by the following command:
#
# pip-compile
#
absl-py==2.1.0
# via
# tensorboard
# tensorflow
aiohttp==3.9.5
# via fsspec
aiosignal==1.3.1
# via aiohttp
alembic==1.13.2
# via optuna
antlr4-python3-runtime==4.9.3
# via omegaconf
appdirs==1.4.4
# via pyhyphen-reloaded
asteroid-filterbanks==0.4.0
# via pyannote-audio
astroid==3.2.4
# via pylint
astunparse==1.6.3
# via tensorflow
async-timeout==4.0.3
# via aiohttp
attrs==23.2.0
# via aiohttp
audioread==3.0.1
# via librosa
av==11.0.0
# via faster-whisper
black==24.4.2
# via -r requirements.in
cachetools==5.4.0
# via google-auth
certifi==2024.7.4
# via requests
cffi==1.16.0
# via soundfile
chardet==5.2.0
# via music21
charset-normalizer==3.3.2
# via requests
click==8.1.7
# via
# black
# nltk
# typer
cloudpickle==3.0.0
# via submitit
colorama==0.4.6
# via
# click
# colorlog
# pylint
# pytest
# tqdm
coloredlogs==15.0.1
# via onnxruntime
colorlog==6.8.2
# via optuna
contourpy==1.2.1
# via matplotlib
crepe==0.0.16
# via -r requirements.in
ctranslate2==4.4.0
# via
# faster-whisper
# whisperx
cycler==0.12.1
# via matplotlib
dataclasses==0.6
# via -r requirements.in
dataclasses-json==0.6.7
# via -r requirements.in
decorator==5.1.1
# via librosa
demucs==4.0.1
# via -r requirements.in
dill==0.3.8
# via pylint
docopt==0.6.2
# via
# num2words
# pyannote-metrics
# pyannote-pipeline
dora-search==0.1.12
# via demucs
einops==0.8.0
# via
# demucs
# pyannote-audio
exceptiongroup==1.2.2
# via pytest
faster-whisper==1.1.0
# via whisperx
ffmpeg-python==0.2.0
# via -r requirements.in
filelock==3.15.4
# via
# huggingface-hub
# pyannote-pipeline
# torch
# transformers
flatbuffers==24.3.25
# via
# onnxruntime
# tensorflow
fonttools==4.53.1
# via matplotlib
frozenlist==1.4.1
# via
# aiohttp
# aiosignal
fsspec[http]==2024.6.1
# via
# huggingface-hub
# lightning
# pytorch-lightning
# torch
future==1.0.0
# via ffmpeg-python
gast==0.4.0
# via tensorflow
google-auth==2.32.0
# via
# google-auth-oauthlib
# tensorboard
google-auth-oauthlib==0.4.6
# via tensorboard
google-pasta==0.2.0
# via tensorflow
greenlet==3.0.3
# via sqlalchemy
grpcio==1.65.1
# via
# tensorboard
# tensorflow
h5py==3.11.0
# via
# crepe
# tensorflow
hmmlearn==0.3.2
# via crepe
huggingface-hub==0.24.2
# via
# faster-whisper
# pyannote-audio
# speechbrain
# tokenizers
# transformers
humanfriendly==10.0
# via coloredlogs
hyperpyyaml==1.2.2
# via speechbrain
idna==3.7
# via
# requests
# yarl
imageio==2.34.2
# via crepe
iniconfig==2.0.0
# via pytest
inputimeout==1.0.4
# via -r requirements.in
isort==5.13.2
# via
# -r requirements.in
# pylint
jinja2==3.1.4
# via torch
joblib==1.4.2
# via
# librosa
# music21
# nltk
# scikit-learn
# speechbrain
jsonpickle==3.2.2
# via music21
julius==0.2.7
# via
# demucs
# torch-audiomentations
keras==2.10.0
# via tensorflow
keras-preprocessing==1.1.2
# via tensorflow
kiwisolver==1.4.5
# via matplotlib
lameenc==1.7.0
# via demucs
langcodes==3.5.0
# via -r requirements.in
language-data==1.2.0
# via langcodes
levenshtein==0.25.1
# via python-levenshtein
libclang==18.1.1
# via tensorflow
librosa==0.9.2
# via
# -r requirements.in
# torch-audiomentations
lightning==2.3.3
# via pyannote-audio
lightning-utilities==0.11.6
# via
# lightning
# pytorch-lightning
# torchmetrics
llvmlite==0.43.0
# via numba
mako==1.3.5
# via alembic
marisa-trie==1.2.0
# via language-data
markdown==3.6
# via tensorboard
markdown-it-py==3.0.0
# via rich
markupsafe==2.1.5
# via
# jinja2
# mako
# werkzeug
marshmallow==3.21.3
# via dataclasses-json
matplotlib==3.10.0
# via
# -r requirements.in
# crepe
# music21
# pyannote-metrics
mccabe==0.7.0
# via pylint
mdurl==0.1.2
# via markdown-it-py
mido==1.3.2
# via pretty-midi
more-itertools==10.3.0
# via music21
mpmath==1.3.0
# via sympy
multidict==6.0.5
# via
# aiohttp
# yarl
music21==9.3.0
# via -r requirements.in
musicbrainzngs==0.7.1
# via -r requirements.in
mypy-extensions==1.0.0
# via
# black
# typing-inspect
networkx==3.3
# via torch
nltk==3.8.1
# via whisperx
num2words==0.5.14
# via -r requirements.in
numba==0.60.0
# via
# librosa
# resampy
numpy==1.26.4
# via
# -r requirements.in
# asteroid-filterbanks
# contourpy
# crepe
# ctranslate2
# h5py
# hmmlearn
# imageio
# keras-preprocessing
# librosa
# lightning
# matplotlib
# music21
# numba
# onnxruntime
# openunmix
# opt-einsum
# optuna
# pandas
# pretty-midi
# pyannote-core
# pyannote-metrics
# pytorch-lightning
# pytorch-metric-learning
# resampy
# scikit-learn
# scipy
# speechbrain
# tensorboard
# tensorboardx
# tensorflow
# torchmetrics
# transformers
oauthlib==3.2.2
# via requests-oauthlib
omegaconf==2.3.0
# via
# dora-search
# pyannote-audio
onnxruntime==1.18.1
# via faster-whisper
openunmix==1.3.0
# via demucs
opt-einsum==3.3.0
# via tensorflow
optuna==3.6.1
# via pyannote-pipeline
packaging==23.2
# via
# -r requirements.in
# black
# huggingface-hub
# librosa
# lightning
# lightning-utilities
# marshmallow
# matplotlib
# mido
# onnxruntime
# optuna
# pooch
# pytest
# pytorch-lightning
# speechbrain
# tensorboardx
# tensorflow
# torch-pitch-shift
# torchmetrics
# transformers
pandas==2.2.2
# via
# pyannote-database
# pyannote-metrics
# whisperx
pathspec==0.12.1
# via black
pillow==10.4.0
# via
# imageio
# matplotlib
platformdirs==4.2.2
# via
# black
# pooch
# pylint
pluggy==1.5.0
# via pytest
pooch==1.8.2
# via librosa
pretty-midi==0.2.10
# via -r requirements.in
primepy==1.3
# via torch-pitch-shift
protobuf==3.19.6
# via
# onnxruntime
# tensorboard
# tensorboardx
# tensorflow
pyannote-audio==3.3.2
# via whisperx
pyannote-core==5.0.0
# via
# pyannote-audio
# pyannote-database
# pyannote-metrics
# pyannote-pipeline
pyannote-database==5.1.0
# via
# pyannote-audio
# pyannote-metrics
# pyannote-pipeline
pyannote-metrics==3.2.1
# via pyannote-audio
pyannote-pipeline==3.0.1
# via pyannote-audio
pyasn1==0.6.0
# via
# pyasn1-modules
# rsa
pyasn1-modules==0.4.0
# via google-auth
pycparser==2.22
# via cffi
pydub==0.25.1
# via -r requirements.in
pygments==2.18.0
# via rich
pyhyphen-reloaded==4.0.6
# via -r requirements.in
pylint==3.2.6
# via -r requirements.in
pyparsing==3.1.2
# via matplotlib
pyreadline3==3.4.1
# via humanfriendly
pytest==8.3.2
# via -r requirements.in
python-dateutil==2.9.0.post0
# via
# matplotlib
# pandas
python-levenshtein==0.25.1
# via -r requirements.in
pytorch-lightning==2.3.3
# via lightning
pytorch-metric-learning==2.6.0
# via pyannote-audio
pytz==2024.1
# via pandas
pyyaml==6.0.1
# via
# ctranslate2
# demucs
# huggingface-hub
# hyperpyyaml
# lightning
# omegaconf
# optuna
# pyannote-database
# pyannote-pipeline
# pytorch-lightning
# transformers
rapidfuzz==3.9.4
# via levenshtein
regex==2024.7.24
# via
# nltk
# transformers
requests==2.32.3
# via
# huggingface-hub
# music21
# pooch
# pyhyphen-reloaded
# requests-oauthlib
# tensorboard
# transformers
requests-oauthlib==2.0.0
# via google-auth-oauthlib
resampy==0.2.2
# via
# crepe
# librosa
retrying==1.3.4
# via dora-search
rich==13.7.1
# via
# pyannote-audio
# typer
rsa==4.9
# via google-auth
ruamel-yaml==0.18.6
# via hyperpyyaml
ruamel-yaml-clib==0.2.8
# via ruamel-yaml
safetensors==0.4.3
# via transformers
scikit-learn==1.5.1
# via
# crepe
# hmmlearn
# librosa
# pyannote-metrics
# pyannote-pipeline
# pytorch-metric-learning
scipy==1.14.1
# via
# -r requirements.in
# crepe
# hmmlearn
# librosa
# pyannote-core
# pyannote-metrics
# resampy
# scikit-learn
# speechbrain
semver==3.0.2
# via pyannote-audio
sentencepiece==0.2.0
# via speechbrain
shellingham==1.5.4
# via typer
six==1.16.0
# via
# astunparse
# google-pasta
# keras-preprocessing
# pretty-midi
# python-dateutil
# resampy
# retrying
# tensorflow
sortedcontainers==2.4.0
# via pyannote-core
soundfile==0.12.1
# via
# librosa
# pyannote-audio
speechbrain==1.0.0
# via pyannote-audio
sqlalchemy==2.0.31
# via
# alembic
# optuna
submitit==1.5.1
# via dora-search
sympy==1.13.1
# via
# onnxruntime
# pyannote-metrics
# torch
tabulate==0.9.0
# via pyannote-metrics
tensorboard==2.10.1
# via tensorflow
tensorboard-data-server==0.6.1
# via tensorboard
tensorboard-plugin-wit==1.8.1
# via tensorboard
tensorboardx==2.6
# via pyannote-audio
tensorflow==2.10.0
# via -r requirements.in
tensorflow-estimator==2.10.0
# via tensorflow
tensorflow-io-gcs-filesystem==0.31.0
# via tensorflow
termcolor==2.4.0
# via tensorflow
threadpoolctl==3.5.0
# via scikit-learn
tokenizers==0.15.2
# via
# faster-whisper
# transformers
tomli==2.0.1
# via
# black
# pylint
# pytest
tomlkit==0.13.0
# via pylint
torch==2.4.0
# via
# asteroid-filterbanks
# demucs
# dora-search
# julius
# lightning
# openunmix
# pyannote-audio
# pytorch-lightning
# pytorch-metric-learning
# speechbrain
# torch-audiomentations
# torch-pitch-shift
# torchaudio
# torchmetrics
# whisperx
torch-audiomentations==0.11.1
# via pyannote-audio
torch-pitch-shift==1.2.4
# via torch-audiomentations
torchaudio==2.4.0
# via
# demucs
# openunmix
# pyannote-audio
# speechbrain
# torch-audiomentations
# torch-pitch-shift
# whisperx
torchmetrics==1.4.0.post0
# via
# lightning
# pyannote-audio
# pytorch-lightning
tqdm==4.67.1
# via
# -r requirements.in
# demucs
# faster-whisper
# huggingface-hub
# lightning
# nltk
# openunmix
# optuna
# pyannote-pipeline
# pytorch-lightning
# pytorch-metric-learning
# speechbrain
# transformers
transformers==4.39.3
# via whisperx
treetable==0.2.5
# via dora-search
typer==0.12.3
# via pyannote-database
typing-extensions==4.12.2
# via
# alembic
# asteroid-filterbanks
# astroid
# black
# huggingface-hub
# lightning
# lightning-utilities
# pyannote-core
# pytorch-lightning
# sqlalchemy
# submitit
# tensorflow
# torch
# typer
# typing-inspect
typing-inspect==0.9.0
# via dataclasses-json
tzdata==2024.1
# via pandas
unidecode==1.3.8
# via -r requirements.in
urllib3==2.2.2
# via requests
webcolors==24.6.0
# via music21
werkzeug==3.0.3
# via tensorboard
wheel==0.43.0
# via
# astunparse
# pyhyphen-reloaded
# tensorboard
whisperx @ git+https://github.com/m-bain/whisperx.git
# via -r requirements.in
wrapt==1.16.0
# via tensorflow
yarl==1.9.4
# via aiohttp
yt-dlp==2024.12.23
# via -r requirements.in
# The following packages are considered to be unsafe in a requirements file:
# setuptools
+21 -1
View File
@@ -1,2 +1,22 @@
#!/bin/bash
gnome-terminal -- bash -c ".venv\Scripts\activate; cd src; exec bash"
# Activate virtual environment and run UltraSinger
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
cd "$SCRIPT_DIR"
# Check if .venv exists
if [ ! -d ".venv" ]; then
echo "Error: Virtual environment not found at .venv"
echo "Please run one of the installation scripts first:"
echo " - install/CPU/linux_cpu.sh (for CPU)"
echo " - install/CUDA/linux_cuda_gpu.sh (for GPU with CUDA)"
exit 1
fi
# Activate virtual environment
source .venv/bin/activate
# Navigate to src and start UltraSinger
cd src
echo "Starting UltraSinger..."
python UltraSinger.py
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
# Activate virtual environment and run UltraSinger on macOS
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
cd "$SCRIPT_DIR"
# Check if .venv exists
if [ ! -d ".venv" ]; then
echo "Error: Virtual environment not found at .venv"
echo "Please run the installation script first:"
echo " - install/CPU/macos_cpu.sh"
exit 1
fi
# Activate virtual environment
source .venv/bin/activate
# Navigate to src and start UltraSinger
cd src
echo "Starting UltraSinger..."
python UltraSinger.py
+4 -1
View File
@@ -1,2 +1,5 @@
@echo off
start cmd /k ".venv\Scripts\activate && cd src"
:: Activate the virtual environment and open cmd in the src directory
call .venv\Scripts\activate.bat
cd src
cmd /k
+3 -8
View File
@@ -11,7 +11,7 @@ from modules.Ultrastar.ultrastar_txt import FormatVersion
@dataclass
class Settings:
APP_VERSION = "0.0.13-dev6"
APP_VERSION = "0.0.13.dev16"
CONFIDENCE_THRESHOLD = 0.6
CONFIDENCE_PROMPT_TIMEOUT = 4
@@ -26,6 +26,7 @@ class Settings:
keep_cache = False
interactive_mode = False
user_ffmpeg_path = ""
quantize_to_key = True # Quantize notes to the detected key of the song
# Process data Paths
input_file_path = ""
@@ -46,16 +47,10 @@ class Settings:
whisper_compute_type = None # change to "int8" if low on GPU mem (may reduce accuracy)
keep_numbers = False
# Pitch
crepe_model_capacity = "full" # tiny|small|medium|large|full
crepe_step_size = 10 # in miliseconds
# Device
pytorch_device = 'cpu' # cpu|cuda
tensorflow_device = 'cpu' # cpu|cuda
force_cpu = False
force_whisper_cpu = False
force_crepe_cpu = False
# MuseScore
musescore_path = None
@@ -65,7 +60,7 @@ class Settings:
# UltraSinger Evaluation Configuration
test_songs_input_folder = None
cache_override_path = None
cache_override_path = None #"C:\\UltraSinger\\test_output"
skip_cache_vocal_separation = False
skip_cache_denoise_vocal_audio = False
skip_cache_transcription = False
+205 -52
View File
@@ -5,6 +5,7 @@ import getopt
import os
import sys
import Levenshtein
import librosa
from packaging import version
@@ -16,9 +17,10 @@ from modules.Audio.vocal_chunks import (
create_audio_chunks_from_transcribed_data,
create_audio_chunks_from_ultrastar_data,
)
from modules.Audio.key_detector import detect_key_from_audio, get_allowed_notes_for_key
from modules.Audio.silence_processing import remove_silence_from_transcription_data, mute_no_singing_parts
from modules.Audio.separation import DemucsModel
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_audio_format
from modules.Audio.youtube import (
download_from_youtube,
)
@@ -41,7 +43,7 @@ from modules.Midi.midi_creator import (
from modules.Midi.MidiSegment import MidiSegment
from modules.Midi.note_length_calculator import get_thirtytwo_note_second, get_sixteenth_note_second
from modules.Pitcher.pitcher import (
get_pitch_with_crepe_file,
get_pitch_with_file,
)
from modules.Pitcher.pitched_data import PitchedData
from modules.Speech_Recognition.TranscriptionResult import TranscriptionResult
@@ -67,7 +69,12 @@ from modules.sheet import create_sheet
from modules.ProcessData import ProcessData, ProcessDataPaths, MediaInfo
from modules.DeviceDetection.device_detection import check_gpu_support
from modules.Image.image_helper import save_image
from modules.ffmpeg_helper import is_ffmpeg_available, get_ffmpeg_and_ffprobe_paths
from modules.ffmpeg_helper import (
is_ffmpeg_available,
get_ffmpeg_and_ffprobe_paths,
is_video_file,
separate_audio_video,
)
from Settings import Settings
@@ -150,6 +157,8 @@ def run() -> tuple[str, Score, Score]:
print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Vocals will not be separated')}")
if not settings.hyphenation:
print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Hyphenation will not be applied')}")
if settings.quantize_to_key:
print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Notes will be quantized to the detected musical key')}")
process_data = InitProcessData()
@@ -162,13 +171,23 @@ def run() -> tuple[str, Score, Score]:
# Create process audio
process_data.process_data_paths.processing_audio_path = CreateProcessAudio(process_data)
# Get BPM from wav file
if not settings.input_file_is_ultrastar_txt:
process_data.media_info.bpm = get_bpm_from_file(process_data.process_data_paths.processing_audio_path)
# Detect key
detected_key, detected_mode = detect_key_from_audio(process_data.process_data_paths.processing_audio_path)
if process_data.media_info.music_key is None:
process_data.media_info.music_key = f"{detected_key} {detected_mode}"
# Audio transcription
process_data.media_info.language = settings.language
if not settings.ignore_audio:
TranscribeAudio(process_data)
# Split syllables into segments
process_data.transcribed_data = split_syllables_into_segments(process_data.transcribed_data,
if not settings.ignore_audio:
process_data.transcribed_data = split_syllables_into_segments(process_data.transcribed_data,
process_data.media_info.bpm)
# Create audio chunks
@@ -178,17 +197,25 @@ def run() -> tuple[str, Score, Score]:
# Pitch audio
process_data.pitched_data = pitch_audio(process_data.process_data_paths)
# Allowed keys for quantization
allowed_notes_for_key = None
if settings.quantize_to_key and not settings.ignore_audio:
allowed_notes_for_key = get_allowed_notes_for_key(detected_key, detected_mode)
# Create Midi_Segments
if not settings.ignore_audio:
process_data.midi_segments = create_midi_segments_from_transcribed_data(process_data.transcribed_data,
process_data.pitched_data)
process_data.midi_segments = create_midi_segments_from_transcribed_data(
process_data.transcribed_data,
process_data.pitched_data,
allowed_notes_for_key
)
else:
process_data.midi_segments = create_repitched_midi_segments_from_ultrastar_txt(process_data.pitched_data,
process_data.parsed_file)
# Merge syllable segments
#tuple[list[MidiSegment], list[TranscribedData]]:
process_data.midi_segments, process_data.transcribed_data = merge_syllable_segments(process_data.midi_segments,
if not settings.ignore_audio:
process_data.midi_segments, process_data.transcribed_data = merge_syllable_segments(process_data.midi_segments,
process_data.transcribed_data,
process_data.media_info.bpm)
@@ -221,14 +248,21 @@ def run() -> tuple[str, Score, Score]:
def split_syllables_into_segments(
transcribed_data: list[TranscribedData],
real_bpm: float) -> list[TranscribedData]:
"""Split every syllable into sub-segments"""
"""Split every syllable into sub-segments
This splits long syllables (including hyphenated ones) into smaller segments
to allow detect for pitch changes within a syllable (e.g., singing a scale).
"""
syllable_segment_size = get_sixteenth_note_second(real_bpm)
thirtytwo_note = get_thirtytwo_note_second(real_bpm)
segment_size_decimal_points = len(str(syllable_segment_size).split(".")[1])
new_data = []
for i, data in enumerate(transcribed_data):
duration = data.end - data.start
# If duration is less than or equal to a 16th note, don't split
if duration <= syllable_segment_size:
new_data.append(data)
continue
@@ -259,7 +293,9 @@ def split_syllables_into_segments(
segment.is_word_end = False
new_data.append(segment)
if partial_segment >= 0.01:
# Only add a partial_segment if it's at least as long as a 32nd note
# Otherwise add it to the last note
if partial_segment >= thirtytwo_note:
first_segment.is_hyphen = True
segment = TranscribedData()
segment.word = "~"
@@ -270,6 +306,9 @@ def split_syllables_into_segments(
segment.is_hyphen = True
segment.is_word_end = False
new_data.append(segment)
elif full_segments >= 1 or len(new_data) > 0:
# Add remaining time to the last note
new_data[-1].end += partial_segment
if has_space:
new_data[-1].word += " "
@@ -280,10 +319,17 @@ def split_syllables_into_segments(
def merge_syllable_segments(midi_segments: list[MidiSegment],
transcribed_data: list[TranscribedData],
real_bpm: float) -> tuple[list[MidiSegment], list[TranscribedData]]:
"""Merge sub-segments of a syllable where the pitch is the same"""
"""Merge sub-segments of a syllable where the pitch is the same
This function handles three cases:
1. Merge consecutive ~ segments with the SAME pitch (same note held)
2. Detect and merge SLIDES: short ~ segments with ±1-2 semitone jumps between syllables
3. Merge ANY consecutive segments (including regular syllables) with the SAME pitch into one word
"""
thirtytwo_note = get_thirtytwo_note_second(real_bpm)
sixteenth_note = get_sixteenth_note_second(real_bpm)
# Slides are typically very short (1-2 16th notes)
max_slide_duration = sixteenth_note * 2
new_data = []
new_midi_notes = []
@@ -291,26 +337,104 @@ def merge_syllable_segments(midi_segments: list[MidiSegment],
previous_data = None
for i, data in enumerate(transcribed_data):
is_note_short = data.end - data.start < thirtytwo_note
is_same_note = midi_segments[i].note == midi_segments[i - 1].note
# Check if previous element exists
is_same_note = i > 0 and midi_segments[i].note == midi_segments[i - 1].note
has_breath_pause = False
if previous_data is not None:
has_breath_pause = data.start - previous_data.end > sixteenth_note
has_breath_pause = (data.start - previous_data.end) > sixteenth_note
if (str(data.word).startswith("~")
and previous_data is not None
and (is_note_short or is_same_note)
and not has_breath_pause):
# Check if current word is a ~ segment (continuation marker)
current_word_stripped = str(data.word).strip()
is_tilde_segment = current_word_stripped == "~"
# Slide detection: Detect short ~ segments with small pitch jumps
is_potential_slide = False
if i > 0 and is_tilde_segment:
duration = data.end - data.start
# Calculate pitch jump in semitones
try:
prev_midi = librosa.note_to_midi(midi_segments[i - 1].note)
curr_midi = librosa.note_to_midi(midi_segments[i].note)
semitone_diff = abs(curr_midi - prev_midi)
# Slide: Short duration AND small pitch jump (1-2 semitones)
is_potential_slide = (duration <= max_slide_duration and
semitone_diff <= 2 and
semitone_diff > 0)
except:
# Ignore slide detection on error
pass
# Check if current segment should be merged with previous due to same pitch
should_merge_same_pitch = False
if (i > 0 and
previous_data is not None and
not is_tilde_segment and # Not a ~ segment
is_same_note and
not has_breath_pause and
not previous_data.is_word_end): # Don't merge across word boundaries
should_merge_same_pitch = True
should_merge_tilde = (is_tilde_segment and
previous_data is not None and
(is_same_note or is_potential_slide) and
not has_breath_pause)
if should_merge_tilde:
new_data[-1].end = data.end
new_midi_notes[-1].end = data.end
# For slides: Keep the original note (not the transition note)
if is_potential_slide and not is_same_note:
new_midi_notes[-1].note = midi_segments[i - 1].note
# Take over space and word_end flag from current segment
# "~ " means end of word - add space to previous segment
if str(data.word).endswith(" "):
new_data[-1].word += " "
new_midi_notes[-1].word += " "
if not new_data[-1].word.endswith(" "):
new_data[-1].word += " "
if not new_midi_notes[-1].word.endswith(" "):
new_midi_notes[-1].word += " "
new_data[-1].is_word_end = True
elif should_merge_same_pitch:
# Merge regular syllable with previous syllable (same pitch)
new_data[-1].end = data.end
new_midi_notes[-1].end = data.end
# Check if current word has space at end before stripping
has_space = str(data.word).endswith(" ")
# Concatenate the words (remove trailing space from previous, add current word)
prev_word = new_data[-1].word.rstrip()
curr_word = data.word.rstrip()
# Remove ~ from BOTH previous and current word if present
if prev_word == "~":
prev_word = ""
if curr_word.startswith("~"):
curr_word = curr_word[1:]
# Concatenate
new_data[-1].word = prev_word + curr_word
new_midi_notes[-1].word = prev_word + curr_word
# Preserve space at end if current segment had it
if has_space:
new_data[-1].word += " "
new_midi_notes[-1].word += " "
if data.is_word_end:
new_data[-1].is_word_end = True
new_data[-1].is_hyphen = False
else:
# Keep hyphen status if either segment was hyphenated
new_data[-1].is_hyphen = new_data[-1].is_hyphen or data.is_hyphen
else:
# Add as new segment
new_data.append(data)
new_midi_notes.append(midi_segments[i])
@@ -340,10 +464,12 @@ def InitProcessData():
settings.output_folder_path,
audio_file_path,
ultrastar_class,
audio_extension,
) = parse_ultrastar_txt(settings.input_file_path, settings.output_folder_path)
process_data = from_ultrastar_txt(ultrastar_class)
process_data.basename = basename
process_data.process_data_paths.audio_output_file_path = audio_file_path
process_data.media_info.audio_extension = audio_extension
# todo: ignore transcribe
settings.ignore_audio = True
@@ -358,7 +484,7 @@ def InitProcessData():
process_data.media_info
) = download_from_youtube(settings.input_file_path, settings.output_folder_path, settings.cookiefile)
else:
# Audio File
# Audio/Video File
print(f"{ULTRASINGER_HEAD} {gold_highlighted('Full Automatic Mode')}")
process_data = ProcessData()
(
@@ -366,7 +492,7 @@ def InitProcessData():
settings.output_folder_path,
process_data.process_data_paths.audio_output_file_path,
process_data.media_info,
) = infos_from_audio_input_file()
) = infos_from_audio_video_input_file()
return process_data
@@ -397,15 +523,15 @@ def CreateUltraStarTxt(process_data: ProcessData):
# Move instrumental and vocals
if settings.create_karaoke and version.parse(settings.format_version.value) < version.parse(
FormatVersion.V1_1_0.value):
karaoke_output_path = os.path.join(settings.output_folder_path, process_data.basename + " [Karaoke].mp3")
convert_wav_to_mp3(process_data.process_data_paths.instrumental_audio_file_path, karaoke_output_path)
karaoke_output_path = os.path.join(settings.output_folder_path, process_data.basename + " [Karaoke]." + process_data.media_info.audio_extension)
convert_audio_format(process_data.process_data_paths.instrumental_audio_file_path, karaoke_output_path)
if version.parse(settings.format_version.value) >= version.parse(FormatVersion.V1_1_0.value):
instrumental_output_path = os.path.join(settings.output_folder_path,
process_data.basename + " [Instrumental].mp3")
convert_wav_to_mp3(process_data.process_data_paths.instrumental_audio_file_path, instrumental_output_path)
vocals_output_path = os.path.join(settings.output_folder_path, process_data.basename + " [Vocals].mp3")
convert_wav_to_mp3(process_data.process_data_paths.vocals_audio_file_path, vocals_output_path)
process_data.basename + " [Instrumental]." + process_data.media_info.audio_extension)
convert_audio_format(process_data.process_data_paths.instrumental_audio_file_path, instrumental_output_path)
vocals_output_path = os.path.join(settings.output_folder_path, process_data.basename + " [Vocals]." + process_data.media_info.audio_extension)
convert_audio_format(process_data.process_data_paths.vocals_audio_file_path, vocals_output_path)
# Create Ultrastar txt
if not settings.ignore_audio:
@@ -489,15 +615,17 @@ def transcribe_audio(cache_folder_path: str, processing_audio_path: str) -> Tran
transcription_result = None
whisper_align_model_string = None
if settings.transcriber == "whisper":
if not settings.whisper_align_model is None: whisper_align_model_string = settings.whisper_align_model.replace("/", "_")
transcription_config = f"{settings.transcriber}_{settings.whisper_model.value}_{settings.pytorch_device}_{whisper_align_model_string}_{settings.whisper_batch_size}_{settings.whisper_compute_type}_{settings.language}"
if not settings.whisper_align_model is None:
whisper_align_model_string = settings.whisper_align_model.replace("/", "_")
whisper_device = "cpu" if settings.force_whisper_cpu else settings.pytorch_device
transcription_config = f"{settings.transcriber}_{settings.whisper_model.value}_{whisper_device}_{whisper_align_model_string}_{settings.whisper_batch_size}_{settings.whisper_compute_type}_{settings.language}"
transcription_path = os.path.join(cache_folder_path, f"{transcription_config}.json")
cached_transcription_available = check_file_exists(transcription_path)
if settings.skip_cache_transcription or not cached_transcription_available:
transcription_result = transcribe_with_whisper(
processing_audio_path,
settings.whisper_model,
settings.pytorch_device,
whisper_device,
settings.whisper_align_model,
settings.whisper_batch_size,
settings.whisper_compute_type,
@@ -516,8 +644,8 @@ def transcribe_audio(cache_folder_path: str, processing_audio_path: str) -> Tran
return transcription_result
def infos_from_audio_input_file() -> tuple[str, str, str, MediaInfo]:
"""Infos from audio input file"""
def infos_from_audio_video_input_file() -> tuple[str, str, str, MediaInfo]:
"""Infos from audio/video input file"""
basename = os.path.basename(settings.input_file_path)
basename_without_ext = os.path.splitext(basename)[0]
@@ -529,27 +657,52 @@ def infos_from_audio_input_file() -> tuple[str, str, str, MediaInfo]:
song_info = search_musicbrainz(title, artist)
basename_without_ext = f"{song_info.artist} - {song_info.title}"
extension = os.path.splitext(basename)[1]
basename = f"{basename_without_ext}{extension}"
song_folder_output_path = os.path.join(settings.output_folder_path, basename_without_ext)
song_folder_output_path = get_unused_song_output_dir(song_folder_output_path)
os_helper.create_folder(song_folder_output_path)
os_helper.copy(settings.input_file_path, song_folder_output_path)
os_helper.rename(
os.path.join(song_folder_output_path, os.path.basename(settings.input_file_path)),
os.path.join(song_folder_output_path, basename),
)
extension = os.path.splitext(basename)[1]
if is_video_file(settings.input_file_path):
print(f"{ULTRASINGER_HEAD} Video file detected - separating audio and video")
video_with_audio_basename = f"{basename_without_ext}{extension}"
video_with_audio_path = os.path.join(song_folder_output_path, video_with_audio_basename)
os_helper.copy(settings.input_file_path, video_with_audio_path)
# Separate audio and video
ultrastar_audio_input_path, final_video_path, audio_ext, video_ext = separate_audio_video(
video_with_audio_path, basename_without_ext, song_folder_output_path
)
else:
# Audio file
basename_with_ext = f"{basename_without_ext}{extension}"
audio_ext = extension.lstrip('.')
video_ext = None
os_helper.copy(settings.input_file_path, song_folder_output_path)
os_helper.rename(
os.path.join(song_folder_output_path, os.path.basename(settings.input_file_path)),
os.path.join(song_folder_output_path, basename_with_ext),
)
ultrastar_audio_input_path = os.path.join(song_folder_output_path, basename_with_ext)
# Todo: Read ID3 tags
if song_info.cover_image_data is not None:
save_image(song_info.cover_image_data, basename_without_ext, song_folder_output_path)
ultrastar_audio_input_path = os.path.join(song_folder_output_path, basename)
real_bpm = get_bpm_from_file(settings.input_file_path)
return (
basename_without_ext,
song_folder_output_path,
ultrastar_audio_input_path,
MediaInfo(artist=song_info.artist, title=song_info.title, year=song_info.year, genre=song_info.genres, bpm=real_bpm, cover_url=song_info.cover_url),
MediaInfo(
artist=song_info.artist,
title=song_info.title,
year=song_info.year,
genre=song_info.genres,
cover_url=song_info.cover_url,
audio_extension=audio_ext,
video_extension=video_ext
),
)
@@ -557,16 +710,13 @@ def pitch_audio(
process_data_paths: ProcessDataPaths) -> PitchedData:
"""Pitch audio"""
pitching_config = f"crepe_{settings.ignore_audio}_{settings.crepe_model_capacity}_{settings.crepe_step_size}_{settings.tensorflow_device}"
pitching_config = f"swiftf0_{settings.ignore_audio}"
pitched_data_path = os.path.join(process_data_paths.cache_folder_path, f"{pitching_config}.json")
cache_available = check_file_exists(pitched_data_path)
if settings.skip_cache_transcription or not cache_available:
pitched_data = get_pitch_with_crepe_file(
process_data_paths.processing_audio_path,
settings.crepe_model_capacity,
settings.crepe_step_size,
settings.tensorflow_device,
if settings.skip_cache_pitch_detection or not cache_available:
pitched_data = get_pitch_with_file(
process_data_paths.processing_audio_path
)
pitched_data_json = pitched_data.to_json()
@@ -594,7 +744,7 @@ def main(argv: list[str]) -> None:
def check_requirements() -> None:
if not settings.force_cpu:
settings.tensorflow_device, settings.pytorch_device = check_gpu_support()
settings.pytorch_device = check_gpu_support()
print(f"{ULTRASINGER_HEAD} ----------------------")
if not is_ffmpeg_available(settings.user_ffmpeg_path):
@@ -701,6 +851,8 @@ def init_settings(argv: list[str]) -> Settings:
settings.cookiefile = arg
elif opt in ("--interactive"):
settings.interactive_mode = True
elif opt in ("--quantize_to_key"):
settings.quantize_to_key = arg
elif opt in ("--ffmpeg"):
settings.user_ffmpeg_path = arg
if settings.output_folder_path == "":
@@ -741,6 +893,7 @@ def arg_options():
"keep_cache",
"musescore_path=",
"keep_numbers",
"quantize_to_key",
"interactive",
"cookiefile=",
"ffmpeg="
+9 -2
View File
@@ -1,4 +1,5 @@
import librosa
import soundfile as sf
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted
@@ -6,7 +7,7 @@ from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted
def get_bpm_from_data(data, sampling_rate):
"""Get real bpm from audio data"""
onset_env = librosa.onset.onset_strength(y=data, sr=sampling_rate)
wav_tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sampling_rate)
wav_tempo = librosa.feature.tempo(onset_envelope=onset_env, sr=sampling_rate)
print(f"{ULTRASINGER_HEAD} BPM is {blue_highlighted(str(round(wav_tempo[0], 2)))}")
return wav_tempo[0]
@@ -14,5 +15,11 @@ def get_bpm_from_data(data, sampling_rate):
def get_bpm_from_file(wav_file: str) -> float:
"""Get real bpm from audio file"""
data, sampling_rate = librosa.load(wav_file, sr=None)
data, sampling_rate = sf.read(wav_file, dtype='float32')
# Transpose if stereo to match librosa's expected format
if len(data.shape) > 1:
data = data.T
# Convert to mono if stereo
if data.ndim > 1:
data = librosa.to_mono(data)
return get_bpm_from_data(data, sampling_rate)
+25 -6
View File
@@ -1,6 +1,7 @@
"""Convert audio to other formats"""
from pydub import AudioSegment
import subprocess
import os
import librosa
import soundfile as sf
@@ -14,9 +15,27 @@ def convert_audio_to_mono_wav(input_file_path: str, output_file_path: str) -> No
sf.write(output_file_path, y, sr)
def convert_wav_to_mp3(input_file_path: str, output_file_path: str) -> None:
"""Convert wav to mp3"""
print(f"{ULTRASINGER_HEAD} Converting wav to mp3. -> {output_file_path}")
def convert_audio_format(input_file_path: str, output_file_path: str) -> None:
"""Convert audio to the format specified by the output file extension using ffmpeg"""
output_format = os.path.splitext(output_file_path)[1].lstrip('.')
sound = AudioSegment.from_wav(input_file_path)
sound.export(output_file_path, format="mp3")
print(f"{ULTRASINGER_HEAD} Converting audio to {output_format}. -> {output_file_path}")
# todo: makes it sense to reencode here? Its only used for Instrumental and Vocal
# Use ffmpeg for audio conversion
# -i: input file
# -y: overwrite output file without asking
# -loglevel error: only show errors
# -q:a 0: best quality for VBR formats (mp3, ogg)
# -codec:a copy would be fastest but only works if formats match
cmd = [
"ffmpeg",
"-i", input_file_path,
"-y",
"-loglevel", "error",
"-q:a", "0",
output_file_path
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
raise RuntimeError(f"FFmpeg audio conversion failed: {result.stderr}")
+147
View File
@@ -0,0 +1,147 @@
"""Key detection and pitch quantization to musical scale"""
import librosa
import numpy as np
from modules.console_colors import (
ULTRASINGER_HEAD,
blue_highlighted)
# scales (in semitones relative to root note)
MAJOR_SCALE = [0, 2, 4, 5, 7, 9, 11]
MINOR_SCALE = [0, 2, 3, 5, 7, 8, 10]
NOTE_NAMES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B']
def detect_key_from_audio(audio_path: str) -> tuple[str, str]:
"""
Detect the key and mode (major/minor) of a song from audio file.
Args:
audio_path: Path to audio file
Returns:
Tuple of (key_note, mode) e.g., ('C', 'major') or ('A', 'minor')
"""
print(f"{ULTRASINGER_HEAD} Detecting musical key")
y, sr = librosa.load(audio_path, sr=None, duration=60.0) # Analyze first 60 seconds
chroma = librosa.feature.chroma_cqt(y=y, sr=sr)
chroma_avg = np.mean(chroma, axis=1)
chroma_avg = chroma_avg / np.sum(chroma_avg)
major_template = np.array([1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1]) # Major scale pattern
minor_template = np.array([1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0]) # Minor scale pattern
major_template = major_template / np.sum(major_template)
minor_template = minor_template / np.sum(minor_template)
best_correlation = -1
best_key = None
best_mode = None
# Try all 12 keys
for shift in range(12):
shifted_major = np.roll(major_template, shift)
shifted_minor = np.roll(minor_template, shift)
major_corr = np.corrcoef(chroma_avg, shifted_major)[0, 1]
minor_corr = np.corrcoef(chroma_avg, shifted_minor)[0, 1]
if major_corr > best_correlation:
best_correlation = major_corr
best_key = NOTE_NAMES[shift]
best_mode = 'major'
if minor_corr > best_correlation:
best_correlation = minor_corr
best_key = NOTE_NAMES[shift]
best_mode = 'minor'
print(f"{ULTRASINGER_HEAD} Detected key: {blue_highlighted(best_key)} {blue_highlighted(best_mode)}")
return best_key, best_mode
def get_allowed_notes_for_key(key_note: str, mode: str) -> set[str]:
"""
Get all allowed note names for a given key across all octaves.
Args:
key_note: Root note (e.g., 'C', 'D#', 'F')
mode: 'major' or 'minor'
Returns:
Set of allowed note names (e.g., {'C', 'D', 'E', ...})
"""
root_idx = NOTE_NAMES.index(key_note)
scale = MAJOR_SCALE if mode == 'major' else MINOR_SCALE
allowed_notes = set()
for interval in scale:
note_idx = (root_idx + interval) % 12
allowed_notes.add(NOTE_NAMES[note_idx])
return allowed_notes
def quantize_note_to_key(note: str, allowed_notes: set[str]) -> str:
"""
Quantize a note to the nearest allowed note in the key.
Args:
note: MIDI note name (e.g., 'C4', 'D#5')
allowed_notes: Set of allowed note names without octave
Returns:
Quantized note name
"""
# Parse note and octave
if len(note) == 0:
return note
# Extract note name and octave
if len(note) >= 2 and note[-2] == '#':
note_name = note[:-1]
octave = note[-1]
else:
note_name = note[:-1] if note[-1].isdigit() else note
octave = note[-1] if note[-1].isdigit() else ''
# If already in key, return as is
if note_name in allowed_notes:
return note
# Find nearest allowed note
try:
note_midi = librosa.note_to_midi(note)
except:
return note
# Try all allowed notes in nearby octaves
min_distance = float('inf')
best_note = note
for allowed_note_name in allowed_notes:
# Try same octave and adjacent octaves
for oct_offset in [-1, 0, 1]:
if not octave:
test_octave = 4 + oct_offset
else:
test_octave = int(octave) + oct_offset
if test_octave < 0 or test_octave > 8:
continue
test_note = f"{allowed_note_name}{test_octave}"
try:
test_midi = librosa.note_to_midi(test_note)
distance = abs(test_midi - note_midi)
if distance < min_distance:
min_distance = distance
best_note = test_note
except:
continue
#print(f"{ULTRASINGER_HEAD} Moved {blue_highlighted(note)} to {blue_highlighted(best_note)}")
return best_note
+32 -28
View File
@@ -10,6 +10,7 @@ from modules.Audio.bpm import get_bpm_from_file
from modules.console_colors import ULTRASINGER_HEAD
from modules.Image.image_helper import save_image
from modules.musicbrainz_client import search_musicbrainz
from modules.ffmpeg_helper import separate_audio_video
def get_youtube_title(url: str, cookiefile: str = None) -> tuple[str, str]:
@@ -30,20 +31,18 @@ def get_youtube_title(url: str, cookiefile: str = None) -> tuple[str, str]:
return result["channel"].strip(), result["title"].strip()
def __download_youtube_audio(url: str, clear_filename: str, output_path: str, cookiefile: str = None):
"""Download audio from YouTube"""
def __download_youtube_video_with_audio(url: str, clear_filename: str, output_path: str, cookiefile: str = None) -> str:
"""Download video with audio from YouTube and return the file extension"""
print(f"{ULTRASINGER_HEAD} Downloading Audio")
print(f"{ULTRASINGER_HEAD} Downloading Video with Audio")
ydl_opts = {
"format": "bestaudio/best",
"outtmpl": output_path + "/" + clear_filename,
"postprocessors": [
{"key": "FFmpegExtractAudio", "preferredcodec": "mp3"}
],
"format": "bestvideo[ext=mp4]+bestaudio/best",
"outtmpl": output_path + "/" + clear_filename + ".%(ext)s",
"merge_output_format": "mp4",
"cookiefile": cookiefile,
}
__start_download(ydl_opts, url)
return "mp4"
def __download_youtube_thumbnail(url: str, clear_filename: str, output_path: str, cookiefile: str = None) -> str:
@@ -75,16 +74,6 @@ def download_and_convert_thumbnail(ydl_opts, url: str, clear_filename: str, outp
return ""
def __download_youtube_video(url: str, clear_filename: str, output_path: str, cookiefile: str = None) -> None:
"""Download video from YouTube"""
print(f"{ULTRASINGER_HEAD} Downloading Video")
ydl_opts = {
"format": "bestvideo[ext=mp4]/mp4",
"outtmpl": output_path + "/" + clear_filename + ".mp4",
"cookiefile": cookiefile,
}
__start_download(ydl_opts, url)
def __start_download(ydl_opts, url: str) -> None:
@@ -104,26 +93,41 @@ def download_from_youtube(input_url: str, output_folder_path: str, cookiefile: s
song_info = search_musicbrainz(title, artist)
basename_without_ext = sanitize_filename(f"{song_info.artist} - {song_info.title}")
basename = basename_without_ext + ".mp3"
song_output = os.path.join(output_folder_path, basename_without_ext)
song_output = get_unused_song_output_dir(song_output)
os_helper.create_folder(song_output)
__download_youtube_audio(input_url, basename_without_ext, song_output, cookiefile)
__download_youtube_video(input_url, basename_without_ext, song_output, cookiefile)
print(f"{ULTRASINGER_HEAD} Downloading from YouTube")
video_ext = __download_youtube_video_with_audio(
input_url, basename_without_ext, song_output, cookiefile
)
video_with_audio_path = os.path.join(song_output, f"{basename_without_ext}.{video_ext}")
# Separate audio and video
audio_file_path, final_video_path, audio_ext, video_ext = separate_audio_video(
video_with_audio_path, basename_without_ext, song_output
)
if song_info.cover_url is not None and song_info.cover_image_data is not None:
cover_url = song_info.cover_url
save_image(song_info.cover_image_data, basename_without_ext, song_output)
else:
cover_url = __download_youtube_thumbnail(
input_url, basename_without_ext, song_output
)
audio_file_path = os.path.join(song_output, basename)
real_bpm = get_bpm_from_file(audio_file_path)
input_url, basename_without_ext, song_output, cookiefile
)
return (
basename_without_ext,
song_output,
audio_file_path,
MediaInfo(artist=song_info.artist, title=song_info.title, year=song_info.year, genre=song_info.genres, bpm=real_bpm,
cover_url=cover_url, video_url=input_url),
MediaInfo(
artist=song_info.artist,
title=song_info.title,
year=song_info.year,
genre=song_info.genres,
cover_url=cover_url,
video_url=input_url,
audio_extension=audio_ext,
video_extension=video_ext
),
)
@@ -1,38 +1,19 @@
"""Device detection module."""
import torch
import os
import tensorflow as tf
from modules.console_colors import ULTRASINGER_HEAD, red_highlighted, blue_highlighted
tensorflow_gpu_supported = False
pytorch_gpu_supported = False
def check_gpu_support() -> tuple[bool, bool]:
"""Check worker device (e.g cuda or cpu) supported by tensorflow and pytorch"""
def check_gpu_support() -> str:
"""Check worker device (e.g cuda or cpu) supported by pytorch"""
print(f"{ULTRASINGER_HEAD} Checking GPU support.")
pytorch_gpu_supported = __check_pytorch_support()
tensorflow_gpu_supported = __check_tensorflow_support()
return 'cuda' if tensorflow_gpu_supported else 'cpu', 'cuda' if pytorch_gpu_supported else 'cpu'
def __check_tensorflow_support():
tensorflow_gpu_supported = False
gpus = tf.config.list_physical_devices('GPU')
if gpus:
tensorflow_gpu_supported = True
print(f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - using {red_highlighted('cuda')} gpu.")
else:
print(
f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - there are no {red_highlighted('cuda')} devices available -> Using {red_highlighted('cpu')}.")
if os.name == 'nt':
print(
f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - versions above 2.10 dropped GPU support for Windows, refer to the readme for possible solutions.")
return tensorflow_gpu_supported
return 'cuda' if pytorch_gpu_supported else 'cpu'
def __check_pytorch_support():
+46 -9
View File
@@ -17,7 +17,7 @@ from modules.console_colors import (
from modules.Ultrastar.ultrastar_txt import UltrastarTxtValue
from modules.Pitcher.pitched_data import PitchedData
from modules.Pitcher.pitched_data_helper import get_frequencies_with_high_confidence
from modules.Audio.key_detector import quantize_note_to_key
def create_midi_instrument(midi_segments: list[MidiSegment]) -> object:
"""Converts an Ultrastar data to a midi instrument"""
@@ -83,9 +83,20 @@ def find_nearest_index(array: list[float], value: float) -> int:
return idx
def create_midi_notes_from_pitched_data(start_times: list[float], end_times: list[float], words: list[str], pitched_data: PitchedData) -> list[
MidiSegment]:
"""Create midi notes from pitched data"""
def create_midi_notes_from_pitched_data(start_times: list[float], end_times: list[float], words: list[str],
pitched_data: PitchedData, allowed_notes: set[str] = None) -> list[MidiSegment]:
"""Create midi notes from pitched data
Args:
start_times: List of start times
end_times: List of end times
words: List of words/syllables
pitched_data: Pitched data containing frequencies and confidence
allowed_notes: Optional set of allowed note names for key quantization
Returns:
List of MidiSegments
"""
print(f"{ULTRASINGER_HEAD} Creating midi_segments")
midi_segments = []
@@ -94,7 +105,7 @@ def create_midi_notes_from_pitched_data(start_times: list[float], end_times: lis
end_time = end_times[index]
word = str(words[index])
midi_segment = create_midi_note_from_pitched_data(start_time, end_time, pitched_data, word)
midi_segment = create_midi_note_from_pitched_data(start_time, end_time, pitched_data, word, allowed_notes)
midi_segments.append(midi_segment)
# todo: Progress?
@@ -102,8 +113,20 @@ def create_midi_notes_from_pitched_data(start_times: list[float], end_times: lis
return midi_segments
def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitched_data: PitchedData, word: str) -> MidiSegment:
"""Create midi note from pitched data"""
def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitched_data: PitchedData, word: str,
allowed_notes: set[str] = None) -> MidiSegment:
"""Create midi note from pitched data
Args:
start_time: Start time of the note
end_time: End time of the note
pitched_data: Pitched data containing frequencies and confidence
word: The word/syllable for this note
allowed_notes: Optional set of allowed note names (without octave) for key quantization
Returns:
One MidiSegment
"""
start = find_nearest_index(pitched_data.times, start_time)
end = find_nearest_index(pitched_data.times, end_time)
@@ -121,10 +144,24 @@ def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitch
note = most_frequent(notes)[0][0]
if allowed_notes is not None:
note = quantize_note_to_key(note, allowed_notes)
return MidiSegment(note, start_time, end_time, word)
def create_midi_segments_from_transcribed_data(transcribed_data: list[TranscribedData], pitched_data: PitchedData) -> list[MidiSegment]:
def create_midi_segments_from_transcribed_data(transcribed_data: list[TranscribedData], pitched_data: PitchedData,
allowed_notes: set[str] = None) -> list[MidiSegment]:
"""Create MIDI segments from transcribed data
Args:
transcribed_data: List of transcribed data segments
pitched_data: Pitched data containing frequencies and confidence
allowed_notes: Optional set of allowed note names for key quantization
Returns:
List of MidiSegments
"""
start_times = []
end_times = []
words = []
@@ -135,7 +172,7 @@ def create_midi_segments_from_transcribed_data(transcribed_data: list[Transcribe
end_times.append(midi_segment.end)
words.append(midi_segment.word)
midi_segments = create_midi_notes_from_pitched_data(start_times, end_times, words,
pitched_data)
pitched_data, allowed_notes)
return midi_segments
+52 -52
View File
@@ -1,41 +1,74 @@
"""Pitcher module"""
import os
import numpy as np
import crepe
from scipy.io import wavfile
from swift_f0 import SwiftF0
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
from modules.Midi.midi_creator import convert_frequencies_to_notes, most_frequent
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted
from modules.Pitcher.pitched_data import PitchedData
from modules.Pitcher.pitched_data_helper import get_frequencies_with_high_confidence
_swift_f0_detector = None
def _get_detector():
"""Lazy initialize SwiftF0 detector"""
global _swift_f0_detector
if _swift_f0_detector is None:
# Initialize for general music/speech (wide frequency range) fmin=46.875, fmax=2093.75
# For speech only: fmin=65, fmax=400
_swift_f0_detector = SwiftF0(fmin=46.875, fmax=2093.75, confidence_threshold=0.9)
return _swift_f0_detector
def get_pitch_with_crepe_file(
filename: str, model_capacity: str, step_size: int = 10, device: str = "cpu"
def get_pitch_with_file(
filename: str
) -> PitchedData:
"""Pitch with crepe"""
"""Pitch detection using SwiftF0"""
print(
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted(device)} as worker"
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('SwiftF0')}"
)
sample_rate, audio = wavfile.read(filename)
return get_pitch_with_crepe(audio, sample_rate, model_capacity, step_size)
# Convert stereo to mono if needed
if len(audio.shape) > 1:
audio = np.mean(audio, axis=1)
# Normalize audio to float32 based on dtype
if audio.dtype == np.uint8:
# uint8: range [0, 255] -> subtract 128 and divide by 128
audio = (audio.astype(np.float32) - 128.0) / 128.0
elif audio.dtype in [np.int16, np.int32, np.int64]:
# Signed integers: use iinfo to get max value and normalize
dtype_info = np.iinfo(audio.dtype)
max_val = max(abs(dtype_info.min), abs(dtype_info.max))
audio = audio.astype(np.float32) / float(max_val)
elif audio.dtype == np.float64:
# float64: cast to float32
audio = audio.astype(np.float32)
elif audio.dtype != np.float32:
# Fallback for other types: assume int16 range
audio = audio.astype(np.float32) / 32768.0
return get_pitch_with_swift_f0(audio, sample_rate)
def get_pitch_with_crepe(
audio, sample_rate: int, model_capacity: str, step_size: int = 10
def get_pitch_with_swift_f0(
audio: np.ndarray, sample_rate: int
) -> PitchedData:
"""Pitch with crepe"""
"""Pitch detection using SwiftF0
# Info: The model is trained on 16 kHz audio, so if the input audio has a different sample rate, it will be first resampled to 16 kHz using resampy inside crepe.
SwiftF0 processes audio at 16kHz with 256-sample hop size internally.
Returns frames at approximately 62.5 ms intervals.
"""
detector = _get_detector()
times, frequencies, confidence, activation = crepe.predict(
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
)
# Detect pitch
result = detector.detect_from_array(audio, sample_rate)
# convert to native float for serialization
confidence = [float(x) for x in confidence]
# Convert to PitchedData format
times = [float(t) for t in result.timestamps]
frequencies = [float(f) for f in result.pitch_hz]
confidence = [float(c) for c in result.confidence]
return PitchedData(times, frequencies, confidence)
@@ -53,39 +86,6 @@ def get_pitched_data_with_high_confidence(
return new_pitched_data
# Todo: Unused
def pitch_each_chunk_with_crepe(directory: str,
crepe_model_capacity: str,
crepe_step_size: int,
tensorflow_device: str) -> list[str]:
"""Pitch each chunk with crepe and return midi notes"""
print(f"{ULTRASINGER_HEAD} Pitching each chunk with {blue_highlighted('crepe')}")
midi_notes = []
for filename in sorted(
[f for f in os.listdir(directory) if f.endswith(".wav")],
key=lambda x: int(x.split("_")[1]),
):
filepath = os.path.join(directory, filename)
# todo: stepsize = duration? then when shorter than "it" it should take the duration. Otherwise there a more notes
pitched_data = get_pitch_with_crepe_file(
filepath,
crepe_model_capacity,
crepe_step_size,
tensorflow_device,
)
conf_f = get_frequencies_with_high_confidence(
pitched_data.frequencies, pitched_data.confidence
)
notes = convert_frequencies_to_notes(conf_f)
note = most_frequent(notes)[0][0]
midi_notes.append(note)
# todo: Progress?
# print(filename + " f: " + str(mean))
return midi_notes
class Pitcher:
"""Docstring"""
+5 -2
View File
@@ -19,17 +19,20 @@ class MediaInfo:
"""Media Info"""
title: str
artist: str
bpm: float
bpm: Optional[float] = None
year: Optional[str] = None
genre: Optional[str] = None
language: Optional[str] = None
cover_url: Optional[str] = None
video_url: Optional[str] = None
music_key: Optional[str] = None # (e.g., "C major", "A minor")
video_extension: Optional[str] = None # (e.g., "mp4", "mkv")
audio_extension: Optional[str] = None # (e.g., "mp3", "m4a")
@dataclass
class ProcessData:
"""Data for processing"""
process_data_paths: ProcessDataPaths = ProcessDataPaths()
process_data_paths: ProcessDataPaths = field(default_factory=ProcessDataPaths)
basename: Optional[str] = None
media_info: Optional[MediaInfo] = None
transcribed_data: Optional[List[TranscribedData]] = field(default_factory=list)
+34 -55
View File
@@ -1,7 +1,26 @@
"""Whisper Speech Recognition Module"""
import os
import inspect
import textwrap
# Set environment variable to handle cuDNN loading issues
# CUDA_MODULE_LOADING=LAZY allows PyTorch to continue even if some cuDNN modules are not found
#os.environ['CUDA_MODULE_LOADING'] = 'LAZY'
import torch
# Fix for PyTorch 2.6+ compatibility with omegaconf and old models
# MUST be done before importing whisperx or any other modules that use torch.load
# Monkey-patch torch.load to use weights_only=False for compatibility with older models
# This is necessary because PyTorch 2.6+ changed the default to weights_only=True
# but WhisperX/Pyannote models were saved with pickle and require weights_only=False
_original_torch_load = torch.load
def _patched_torch_load(*args, **kwargs):
# Force weights_only=False regardless of what was passed
kwargs['weights_only'] = False
return _original_torch_load(*args, **kwargs)
torch.load = _patched_torch_load
import whisperx
from enum import Enum
from torch.cuda import OutOfMemoryError
@@ -80,68 +99,15 @@ def transcribe_with_whisper(
keep_numbers: bool = False,
) -> TranscriptionResult:
"""Transcribe with whisper"""
# Info: Monkey Patch FasterWhisperPipeline.detect_language to include error handling for low confidence
src = textwrap.dedent(inspect.getsource(whisperx.asr.FasterWhisperPipeline.detect_language))
# Replace the relevant part of the method
start_token = "if audio.shape[0] < N_SAMPLES:"
end_token = "return language"
replacement = """\
#Added imports
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
from Settings import Settings
from inputimeout import inputimeout, TimeoutOccurred
#End Import addition
if audio.shape[0] < N_SAMPLES:
print("Warning: audio is shorter than 30s, language detection may be inaccurate.")
model_n_mels = self.model.feat_kwargs.get("feature_size")
segment = log_mel_spectrogram(audio[: N_SAMPLES],
n_mels=model_n_mels if model_n_mels is not None else 80,
padding=0 if audio.shape[0] >= N_SAMPLES else N_SAMPLES - audio.shape[0])
encoder_output = self.model.encode(segment)
results = self.model.model.detect_language(encoder_output)
language_token, language_probability = results[0][0]
language = language_token[2:-2]
print(f"Detected language: {language} ({language_probability:.2f}) in first 30s of audio...")
#Added handling for low detection probability
if language_probability < Settings.CONFIDENCE_THRESHOLD:
print(f"{ULTRASINGER_HEAD} {red_highlighted('Warning:')} Language detection probability for detected language {language} is below {Settings.CONFIDENCE_THRESHOLD}, results may be inaccurate.")
print(f"{ULTRASINGER_HEAD} Override the language below or re-run with parameter {blue_highlighted('--language xx')} to specify the song language...")
try:
response = inputimeout(
prompt=f"{ULTRASINGER_HEAD} Do you want to continue with {language} (default) or override with another language (y)? (y/n): ",
timeout=Settings.CONFIDENCE_PROMPT_TIMEOUT
).strip().lower()
except TimeoutOccurred:
import locale
print(f"{ULTRASINGER_HEAD} No user input received in {Settings.CONFIDENCE_PROMPT_TIMEOUT} seconds. Attempting automatic override with system locale.")
print(f"{ULTRASINGER_HEAD} Trying to get language from default locale...")
current_locale = locale.getlocale()
if current_locale[0]:
language_code = current_locale[0][:2].strip().lower()
print(f"{ULTRASINGER_HEAD} Found language code: {language_code} in locale. Setting language to {blue_highlighted(language_code)}...")
language = language_code
else:
print(f"{ULTRASINGER_HEAD} No locale is set.")
response = 'n'
language_response = response == 'y'
if language_response:
language = input(f"{ULTRASINGER_HEAD} Please enter the language code for the language you want to use (e.g. 'en', 'de', 'es', etc.): ").strip().lower()
#End addition
"""
new_src = replace_code_lines(src, start_token, end_token, replacement)
# Compile it and execute it in the target module's namespace
exec(compile(new_src, "<string>", "exec"), whisperx.asr.__dict__)
whisperx.asr.FasterWhisperPipeline.detect_language = whisperx.asr.detect_language
#End Monkey Patch
# Info: Regardless of the audio sampling rate used in the original audio file, whisper resample the audio signal to 16kHz (via ffmpeg). So the standard input from (44.1 or 48 kHz) should work.
print(
f"{ULTRASINGER_HEAD} Loading {blue_highlighted('whisper')} with model {blue_highlighted(model.value)} and {red_highlighted(device)} as worker"
)
if alignment_model is not None:
print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(alignment_model)}")
# Fixme: Why it is not none for cpu?
#compute_type = "int8"
if compute_type is None:
compute_type = "float16" if device == "cuda" else "int8"
@@ -196,6 +162,9 @@ def transcribe_with_whisper(
return TranscriptionResult(transcribed_data, detected_language)
except ValueError as value_error:
# Restore original torch.load in case of error
torch.load = _original_torch_load
if (
"Requested float16 compute type, but the target device or backend do not support efficient float16 computation."
in str(value_error.args[0])
@@ -207,14 +176,24 @@ def transcribe_with_whisper(
raise value_error
except OutOfMemoryError as oom_exception:
# Restore original torch.load in case of error
torch.load = _original_torch_load
print(oom_exception)
print(MEMORY_ERROR_MESSAGE)
raise oom_exception
except Exception as exception:
# Restore original torch.load in case of error
torch.load = _original_torch_load
if "CUDA failed with error out of memory" in str(exception.args[0]):
print(exception)
print(MEMORY_ERROR_MESSAGE)
raise exception
finally:
# Restore original torch.load after models are loaded
# This ensures other modules (like pitch detection) are not affected by the monkey-patch
torch.load = _original_torch_load
def convert_to_transcribed_data(result_aligned):
@@ -66,11 +66,14 @@ def create_ultrastar_txt_from_automation(
ultrastar_txt = UltrastarTxtValue()
ultrastar_txt.version = format_version.value
ultrastar_txt.mp3 = basename + ".mp3"
ultrastar_txt.audio = basename + ".mp3"
ultrastar_txt.vocals = basename + " [Vocals].mp3"
ultrastar_txt.instrumental = basename + " [Instrumental].mp3"
ultrastar_txt.video = basename + ".mp4"
if media_info.audio_extension is None:
raise Exception("Missing Audio extension. It is required to create Ultrastar txt")
ultrastar_txt.mp3 = basename + "." + media_info.audio_extension
ultrastar_txt.audio = basename + "." + media_info.audio_extension
ultrastar_txt.vocals = basename + " [Vocals]." + media_info.audio_extension
ultrastar_txt.instrumental = basename + " [Instrumental]." + media_info.audio_extension
if media_info.video_extension is not None:
ultrastar_txt.video = basename + "." + media_info.video_extension
ultrastar_txt.language = media_info.language
cover = basename + " [CO].jpg"
ultrastar_txt.cover = (
@@ -90,6 +93,9 @@ def create_ultrastar_txt_from_automation(
ultrastar_txt.videoUrl = media_info.video_url
if media_info.cover_url is not None:
ultrastar_txt.coverUrl = media_info.cover_url
if media_info.music_key is not None:
# todo: as list add here?
ultrastar_txt.tags = f"key: {media_info.music_key}"
ultrastar_file_output_path = os.path.join(song_folder_output_path, basename + ".txt")
create_ultrastar_txt(
@@ -101,7 +107,7 @@ def create_ultrastar_txt_from_automation(
if create_karaoke and version.parse(format_version.value) < version.parse(FormatVersion.V1_1_0.value):
title = basename + " [Karaoke]"
ultrastar_txt.title = title
ultrastar_txt.mp3 = title + ".mp3"
ultrastar_txt.mp3 = title + "." + media_info.audio_extension
karaoke_output_path = os.path.join(song_folder_output_path, title)
karaoke_txt_output_path = karaoke_output_path + ".txt"
create_ultrastar_txt(
+19 -18
View File
@@ -33,35 +33,33 @@ def parse(input_file: str) -> UltrastarTxtValue:
for line in txt:
count += 1
if line.startswith("#"):
if line.startswith(f"#{UltrastarTxtTag.ARTIST}"):
if line.startswith(f"#{UltrastarTxtTag.ARTIST.value}"):
ultrastar_class.artist = line.split(":")[1].replace("\n", "")
elif line.startswith(f"#{UltrastarTxtTag.TITLE}"):
elif line.startswith(f"#{UltrastarTxtTag.TITLE.value}"):
ultrastar_class.title = line.split(":")[1].replace("\n", "")
elif line.startswith(f"#{UltrastarTxtTag.MP3}"):
elif line.startswith(f"#{UltrastarTxtTag.MP3.value}"):
ultrastar_class.mp3 = line.split(":")[1].replace("\n", "")
elif line.startswith(f"#{UltrastarTxtTag.AUDIO}"):
elif line.startswith(f"#{UltrastarTxtTag.AUDIO.value}"):
ultrastar_class.audio = line.split(":")[1].replace("\n", "")
elif line.startswith(f"#{UltrastarTxtTag.VIDEO}"):
elif line.startswith(f"#{UltrastarTxtTag.VIDEO.value}"):
ultrastar_class.video = line.split(":")[1].replace("\n", "")
elif line.startswith(f"#{UltrastarTxtTag.GAP}"):
elif line.startswith(f"#{UltrastarTxtTag.GAP.value}"):
ultrastar_class.gap = line.split(":")[1].replace("\n", "")
elif line.startswith(f"#{UltrastarTxtTag.BPM}"):
elif line.startswith(f"#{UltrastarTxtTag.BPM.value}"):
ultrastar_class.bpm = line.split(":")[1].replace("\n", "")
elif line.startswith(f"#{UltrastarTxtTag.VIDEO}"):
ultrastar_class.video = line.split(":")[1].replace("\n", "")
elif line.startswith(f"#{UltrastarTxtTag.VIDEOGAP}"):
elif line.startswith(f"#{UltrastarTxtTag.VIDEOGAP.value}"):
ultrastar_class.videoGap = line.split(":")[1].replace("\n", "")
elif line.startswith(f"#{UltrastarTxtTag.COVER}"):
elif line.startswith(f"#{UltrastarTxtTag.COVER.value}"):
ultrastar_class.cover = line.split(":")[1].replace("\n", "")
elif line.startswith(f"#{UltrastarTxtTag.BACKGROUND}"):
elif line.startswith(f"#{UltrastarTxtTag.BACKGROUND.value}"):
ultrastar_class.background = line.split(":")[1].replace("\n", "")
elif line.startswith(
(
f"{UltrastarTxtNoteTypeTag.FREESTYLE} ",
f"{UltrastarTxtNoteTypeTag.NORMAL} ",
f"{UltrastarTxtNoteTypeTag.GOLDEN} ",
f"{UltrastarTxtNoteTypeTag.RAP} ",
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN} ",
f"{UltrastarTxtNoteTypeTag.FREESTYLE.value} ",
f"{UltrastarTxtNoteTypeTag.NORMAL.value} ",
f"{UltrastarTxtNoteTypeTag.GOLDEN.value} ",
f"{UltrastarTxtNoteTypeTag.RAP.value} ",
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN.value} ",
)
):
parts = line.split()
@@ -86,7 +84,7 @@ def parse(input_file: str) -> UltrastarTxtValue:
return ultrastar_class
def parse_ultrastar_txt(input_file_path: str, output_folder_path: str) -> tuple[str, str, str, UltrastarTxtValue]:
def parse_ultrastar_txt(input_file_path: str, output_folder_path: str) -> tuple[str, str, str, UltrastarTxtValue, str]:
"""Parse Ultrastar txt"""
ultrastar_class = parse(input_file_path)
@@ -100,6 +98,8 @@ def parse_ultrastar_txt(input_file_path: str, output_folder_path: str) -> tuple[
f"an audio file."
)
exit(1)
_, audio_ext_with_dot = os.path.splitext(ultrastar_mp3_name)
audio_ext = audio_ext_with_dot.lstrip('.')
song_output = os.path.join(
output_folder_path,
@@ -118,4 +118,5 @@ def parse_ultrastar_txt(input_file_path: str, output_folder_path: str) -> tuple[
song_output,
str(audio_file_path),
ultrastar_class,
audio_ext
)
@@ -43,13 +43,13 @@ class Points:
def add_point(note_type: str, points: Points) -> Points:
"""Add calculated points to the points object."""
if note_type == UltrastarTxtNoteTypeTag.NORMAL:
if note_type == UltrastarTxtNoteTypeTag.NORMAL.value:
points.notes += 1
elif note_type == UltrastarTxtNoteTypeTag.GOLDEN:
elif note_type == UltrastarTxtNoteTypeTag.GOLDEN.value:
points.golden_notes += 2
elif note_type == UltrastarTxtNoteTypeTag.RAP:
elif note_type == UltrastarTxtNoteTypeTag.RAP.value:
points.rap += 1
elif note_type == UltrastarTxtNoteTypeTag.RAP_GOLDEN:
elif note_type == UltrastarTxtNoteTypeTag.RAP_GOLDEN.value:
points.golden_rap += 2
return points
@@ -75,6 +75,9 @@ def get_score(points: Points) -> Score:
if points.line_bonus == 0
else MAX_SONG_SCORE - MAX_SONG_LINE_BONUS
)
if(points.parts == 0):
print(f"{ULTRASINGER_HEAD} No parts found, returning 0 score")
return score
score.notes = round(
score.max_score * (points.notes + points.rap) / points.parts
)
@@ -100,6 +103,9 @@ def calculate_score(pitched_data: PitchedData, ultrastar_class: UltrastarTxtValu
simple_points = Points()
accurate_points = Points()
if(len(ultrastar_class.UltrastarNoteLines) == 0):
print(f"{ULTRASINGER_HEAD} No note lines found in Ultrastar txt, returning 0 points")
return get_score(simple_points), get_score(accurate_points)
reachable_line_bonus_per_word = MAX_SONG_LINE_BONUS / len(ultrastar_class.UltrastarNoteLines)
step_size = 0.09 # Todo: Whats is the step size of the game? Its not 1/bps -> one beat in seconds s = 60/bpm
+1 -1
View File
@@ -33,7 +33,7 @@ class UltrastarTxtTag(str, Enum):
AUDIO = 'AUDIO' # Its instead of MP3. Just renamed
VOCALS = 'VOCALS' # Vocals only audio
INSTRUMENTAL = 'INSTRUMENTAL' # Instrumental only audio
TAGS = 'TAGS' # Tags for the song. Can be used for filtering
TAGS = 'TAGS' # Tags for the song. Can be used for filtering. Separation with ","
# 1.2.0
VIDEOURL = 'VIDEOURL' # URL to the video file
+34 -33
View File
@@ -53,42 +53,43 @@ def create_ultrastar_txt(
gap = midi_segments[0].start
if version.parse(ultrastar_class.version) >= version.parse("1.0.0"):
file.write(f"#{UltrastarTxtTag.VERSION}:{ultrastar_class.version}\n"),
file.write(f"#{UltrastarTxtTag.ARTIST}:{ultrastar_class.artist}\n")
file.write(f"#{UltrastarTxtTag.TITLE}:{ultrastar_class.title}\n")
file.write(f"#{UltrastarTxtTag.VERSION.value}:{ultrastar_class.version}\n")
file.write(f"#{UltrastarTxtTag.ARTIST.value}:{ultrastar_class.artist}\n")
file.write(f"#{UltrastarTxtTag.TITLE.value}:{ultrastar_class.title}\n")
if ultrastar_class.year is not None:
file.write(f"#{UltrastarTxtTag.YEAR}:{ultrastar_class.year}\n")
file.write(f"#{UltrastarTxtTag.YEAR.value}:{ultrastar_class.year}\n")
if ultrastar_class.language is not None:
file.write(f"#{UltrastarTxtTag.LANGUAGE}:{get_language_name(ultrastar_class.language)}\n")
file.write(f"#{UltrastarTxtTag.LANGUAGE.value}:{get_language_name(ultrastar_class.language)}\n")
if ultrastar_class.genre:
file.write(f"#{UltrastarTxtTag.GENRE}:{ultrastar_class.genre}\n")
file.write(f"#{UltrastarTxtTag.GENRE.value}:{ultrastar_class.genre}\n")
if ultrastar_class.cover is not None:
file.write(f"#{UltrastarTxtTag.COVER}:{ultrastar_class.cover}\n")
file.write(f"#{UltrastarTxtTag.COVER.value}:{ultrastar_class.cover}\n")
if version.parse(ultrastar_class.version) >= version.parse("1.2.0"):
if ultrastar_class.coverUrl is not None:
file.write(f"#{UltrastarTxtTag.COVERURL}:{ultrastar_class.coverUrl}\n")
file.write(f"#{UltrastarTxtTag.COVERURL.value}:{ultrastar_class.coverUrl}\n")
if ultrastar_class.background is not None:
file.write(f"#{UltrastarTxtTag.BACKGROUND}:{ultrastar_class.background}\n")
file.write(f"#{UltrastarTxtTag.MP3}:{ultrastar_class.mp3}\n")
file.write(f"#{UltrastarTxtTag.BACKGROUND.value}:{ultrastar_class.background}\n")
file.write(f"#{UltrastarTxtTag.MP3.value}:{ultrastar_class.mp3}\n")
if version.parse(ultrastar_class.version) >= version.parse("1.1.0"):
file.write(f"#{UltrastarTxtTag.AUDIO}:{ultrastar_class.audio}\n")
file.write(f"#{UltrastarTxtTag.AUDIO.value}:{ultrastar_class.audio}\n")
if ultrastar_class.vocals is not None:
file.write(f"#{UltrastarTxtTag.VOCALS}:{ultrastar_class.vocals}\n")
file.write(f"#{UltrastarTxtTag.VOCALS.value}:{ultrastar_class.vocals}\n")
if ultrastar_class.instrumental is not None:
file.write(f"#{UltrastarTxtTag.INSTRUMENTAL}:{ultrastar_class.instrumental}\n")
if ultrastar_class.tags is not None:
file.write(f"#{UltrastarTxtTag.TAGS}:{ultrastar_class.tags}\n")
file.write(f"#{UltrastarTxtTag.INSTRUMENTAL.value}:{ultrastar_class.instrumental}\n")
if ultrastar_class.video is not None:
file.write(f"#{UltrastarTxtTag.VIDEO}:{ultrastar_class.video}\n")
file.write(f"#{UltrastarTxtTag.VIDEO.value}:{ultrastar_class.video}\n")
if ultrastar_class.videoGap is not None:
file.write(f"#{UltrastarTxtTag.VIDEOGAP}:{ultrastar_class.videoGap}\n")
file.write(f"#{UltrastarTxtTag.VIDEOGAP.value}:{ultrastar_class.videoGap}\n")
if version.parse(ultrastar_class.version) >= version.parse("1.2.0"):
if ultrastar_class.videoUrl is not None:
file.write(f"#{UltrastarTxtTag.VIDEOURL}:{ultrastar_class.videoUrl}\n")
file.write(f"#{UltrastarTxtTag.BPM}:{round(ultrastar_bpm, 2)}\n") # not the real BPM!
file.write(f"#{UltrastarTxtTag.GAP}:{int(gap * 1000)}\n")
file.write(f"#{UltrastarTxtTag.CREATOR}:{ultrastar_class.creator}\n")
file.write(f"#{UltrastarTxtTag.COMMENT}:{ultrastar_class.comment}\n")
file.write(f"#{UltrastarTxtTag.VIDEOURL.value}:{ultrastar_class.videoUrl}\n")
file.write(f"#{UltrastarTxtTag.BPM.value}:{round(ultrastar_bpm, 2)}\n") # not the real BPM!
file.write(f"#{UltrastarTxtTag.GAP.value}:{int(gap * 1000)}\n")
if version.parse(ultrastar_class.version) >= version.parse("1.1.0"):
if ultrastar_class.tags is not None:
file.write(f"#{UltrastarTxtTag.TAGS.value}:{ultrastar_class.tags}\n")
file.write(f"#{UltrastarTxtTag.CREATOR.value}:{ultrastar_class.creator}\n")
file.write(f"#{UltrastarTxtTag.COMMENT.value}:{ultrastar_class.comment}\n")
# Write the singing part
previous_end_beat = 0
@@ -118,7 +119,7 @@ def create_ultrastar_txt(
# 'n2' duration at real beat
# 'n3' pitch where 0 == C4
# 'w' lyric
line = f"{UltrastarTxtNoteTypeTag.NORMAL} " \
line = f"{UltrastarTxtNoteTypeTag.NORMAL.value} " \
f"{str(start_beat)} " \
f"{str(duration)} " \
f"{str(convert_midi_note_to_ultrastar_note(midi_segment))} " \
@@ -140,11 +141,11 @@ def create_ultrastar_txt(
second_to_beat(midi_segment.end - gap, real_bpm)
* multiplication
)
linebreak = f"{UltrastarTxtTag.LINEBREAK} " \
linebreak = f"{UltrastarTxtTag.LINEBREAK.value} " \
f"{str(round(show_next))}\n"
file.write(linebreak)
separated_word_silence = []
file.write(f"{UltrastarTxtTag.FILE_END}")
file.write(f"{UltrastarTxtTag.FILE_END.value}")
def deviation(silence_parts):
@@ -191,7 +192,7 @@ def create_repitched_txt_from_ultrastar_data(
# todo: just add '_repitched' to input_file
with open(output_repitched_ultrastar, "w", encoding=FILE_ENCODING) as file:
for line in txt:
if line.startswith(f"{UltrastarTxtNoteTypeTag.NORMAL} "):
if line.startswith(f"{UltrastarTxtNoteTypeTag.NORMAL.value} "):
parts = re.findall(r"\S+|\s+", line)
# between are whitespaces
# [0] :
@@ -214,21 +215,21 @@ def add_score_to_ultrastar_txt(ultrastar_file_output: str, score: Score) -> None
text = text.split("\n")
for i, line in enumerate(text):
if line.startswith(f"#{UltrastarTxtTag.COMMENT}:"):
if line.startswith(f"#{UltrastarTxtTag.COMMENT.value}:"):
text[
i
] = f"{line} | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}"
break
if line.startswith((
f"{UltrastarTxtNoteTypeTag.FREESTYLE} ",
f"{UltrastarTxtNoteTypeTag.NORMAL} ",
f"{UltrastarTxtNoteTypeTag.GOLDEN} ",
f"{UltrastarTxtNoteTypeTag.RAP} ",
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN} ")):
f"{UltrastarTxtNoteTypeTag.FREESTYLE.value} ",
f"{UltrastarTxtNoteTypeTag.NORMAL.value} ",
f"{UltrastarTxtNoteTypeTag.GOLDEN.value} ",
f"{UltrastarTxtNoteTypeTag.RAP.value} ",
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN.value} ")):
text.insert(
i,
f"#{UltrastarTxtTag.COMMENT}: UltraSinger [GitHub] | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}",
f"#{UltrastarTxtTag.COMMENT.value}: UltraSinger [GitHub] | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}",
)
break
+2 -1
View File
@@ -11,7 +11,7 @@ def print_help() -> None:
[opt]
-h This help text.
-i Ultrastar.txt
audio like .mp3, .wav, youtube link
audio/video like .mp3, .mp4, .wav, youtube link
-o Output folder
[mode]
@@ -54,6 +54,7 @@ def print_help() -> None:
--create_audio_chunks Enable creation of audio chunks. Audio chunks are disabled by default.
--keep_cache Keep cache folder after creation. Cache folder is removed by default.
--plot Enable creation of plots. Plots are disabled by default.
--quantize_to_key Quantize notes to the detected musical key. This removes slides and out-of-key notes.
--format_version 0.3.0|1.0.0|1.1.0|1.2.0 >> ((default) is 1.2.0)
--musescore_path path to MuseScore executable
--keep_numbers Transcribe numbers as digits and not words
+137
View File
@@ -2,6 +2,7 @@
import os
import shutil
import subprocess
def __add_ffmpeg_to_path(ffmpeg_path) -> None:
@@ -29,3 +30,139 @@ def is_ffmpeg_available(user_path: str = "") -> bool:
return True
return False
def extract_audio(input_video_path: str, output_audio_path: str) -> None:
"""Extract audio from video file in original format without re-encoding"""
ffmpeg_path, _ = get_ffmpeg_and_ffprobe_paths()
cmd = [
ffmpeg_path,
"-i",
input_video_path,
"-vn", # No video
"-acodec",
"copy", # Copy audio codec without re-encoding
"-y", # Overwrite output file if exists
output_audio_path,
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
raise Exception(f"FFmpeg audio extraction failed: {result.stderr}")
def remove_audio_from_video(input_video_path: str, output_video_path: str) -> None:
"""Remove audio from video file without re-encoding video"""
ffmpeg_path, _ = get_ffmpeg_and_ffprobe_paths()
cmd = [
ffmpeg_path,
"-i",
input_video_path,
"-an", # No audio
"-vcodec",
"copy", # Copy video codec without re-encoding
"-y", # Overwrite output file if exists
output_video_path,
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
raise Exception(f"FFmpeg audio removal failed: {result.stderr}")
def is_video_file(file_path: str) -> bool:
"""Check if file contains video streams using ffprobe"""
try:
_, ffprobe_path = get_ffmpeg_and_ffprobe_paths()
cmd = [
ffprobe_path,
"-v", "error",
"-select_streams", "v:0",
"-show_entries", "stream=codec_type",
"-of", "default=noprint_wrappers=1:nokey=1",
file_path
]
result = subprocess.run(cmd, capture_output=True, text=True)
return result.returncode == 0 and result.stdout.strip() == "video"
except Exception:
return False
def get_audio_codec_and_extension(video_file_path: str) -> str:
"""
Detect audio codec from video file and return codec name and appropriate file extension.
"""
try:
_, ffprobe_path = get_ffmpeg_and_ffprobe_paths()
cmd = [
ffprobe_path,
"-v", "error",
"-select_streams", "a:0",
"-show_entries", "stream=codec_name",
"-of", "default=noprint_wrappers=1:nokey=1",
video_file_path
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0 or not result.stdout.strip():
# Default to wav if detection fails
return "wav"
codec_name = result.stdout.strip()
codec_to_extension = {
"aac": "aac",
"mp3": "mp3",
"opus": "opus",
"vorbis": "ogg",
"flac": "flac",
"pcm_s16le": "wav",
"pcm_s24le": "wav",
"pcm_s32le": "wav",
"ac3": "ac3",
"eac3": "eac3",
}
extension = codec_to_extension.get(codec_name, "wav")
return extension
except Exception:
# Default to wav if detection fails
return "wav"
def separate_audio_video(video_with_audio_path: str, basename_without_ext: str, output_folder: str) -> tuple[str, str, str, str]:
"""
Separate audio and video from a video file.
Automatically detects the audio codec and uses the appropriate file extension.
"""
from modules.console_colors import ULTRASINGER_HEAD
# Get original video file extension without the dot
_, video_ext_with_dot = os.path.splitext(video_with_audio_path)
video_ext = video_ext_with_dot.lstrip('.')
# Detect audio codec and get appropriate extension
audio_ext = get_audio_codec_and_extension(video_with_audio_path)
print(f"{ULTRASINGER_HEAD} Extracting audio from video")
audio_file_path = os.path.join(output_folder, f"{basename_without_ext}.{audio_ext}")
extract_audio(video_with_audio_path, audio_file_path)
print(f"{ULTRASINGER_HEAD} Creating video without audio")
video_only_path = os.path.join(output_folder, f"{basename_without_ext}_video.{video_ext}")
remove_audio_from_video(video_with_audio_path, video_only_path)
# Remove original video with audio
os.remove(video_with_audio_path)
# Rename video without audio to final name
final_video_path = os.path.join(output_folder, f"{basename_without_ext}.{video_ext}")
os.rename(video_only_path, final_video_path)
return audio_file_path, final_video_path, audio_ext, video_ext
+40 -7
View File
@@ -1,5 +1,7 @@
import musicbrainzngs
import string
import time
from urllib.error import URLError
from Levenshtein import ratio
from dataclasses import dataclass
from typing import Optional
@@ -25,10 +27,22 @@ title_filter = [
]
MAX_RETRIES = 3
def __clean_string(s: str) -> str:
return s.translate(str.maketrans('', '', string.punctuation)).lower().strip()
def __musicbrainz_request(func):
for i in range(MAX_RETRIES):
try:
return func()
except musicbrainzngs.musicbrainz.NetworkError:
time.sleep(1)
return None
def search_musicbrainz(title: str, artist) -> SongInfo:
# Musicbrainz API documentation
# https://python-musicbrainzngs.readthedocs.io/en/latest/api/
@@ -68,8 +82,12 @@ def __single_line_search(search_string):
search_string = __clean_string(search_string)
search_string = __filter_words(search_string)
artists = musicbrainzngs.search_artists(search_string, limit=10, artist=search_string)
recordings = musicbrainzngs.search_recordings(search_string, limit=100, artistname=search_string)
artists = __musicbrainz_request(lambda: musicbrainzngs.search_artists(search_string, limit=10, artist=search_string))
recordings = __musicbrainz_request(lambda: musicbrainzngs.search_recordings(search_string, limit=100, artistname=search_string))
if artists is None or recordings is None:
return None
found_artist = None
for record in recordings['recording-list']:
@@ -114,8 +132,13 @@ def __multi_line_search(artist: str, title: str):
artist1, title1 = artist, title
artist2, title2 = title, artist
result1 = musicbrainzngs.search_recordings(recording=title1, limit=10, artist=artist1, artistname=artist1)
result2 = musicbrainzngs.search_recordings(recording=title2, limit=10, artist=artist2, artistname=artist2)
result1 = __musicbrainz_request(lambda: musicbrainzngs.search_recordings(recording=title1, limit=10, artist=artist1, artistname=artist1))
result2 = __musicbrainz_request(lambda: musicbrainzngs.search_recordings(recording=title2, limit=10, artist=artist2, artistname=artist2))
if result1 is None:
result1 = {'recording-count': 0, 'recording-list': []}
if result2 is None:
result2 = {'recording-count': 0, 'recording-list': []}
# Filter result to ['artist-credit-phrase'] == artist
record1 = [x for x in result1['recording-list'] if
@@ -157,8 +180,14 @@ def __get_image(recording) -> (bytes, str):
if 'release-list' in recording:
for release in recording['release-list']:
try:
image_data = musicbrainzngs.get_image_front(release['id'])
image_list = musicbrainzngs.get_image_list(release['id'])
image_data = __musicbrainz_request(lambda: musicbrainzngs.get_image_front(release['id']))
if image_data is None:
continue
image_list = __musicbrainz_request(lambda: musicbrainzngs.get_image_list(release['id']))
if image_list is None:
continue
for image in image_list['images']:
if image['front']:
image_url = image['image']
@@ -179,7 +208,11 @@ def __get_year(recording):
return year
release_group_id = recording['release-list'][0]['release-group']['id']
release_group = musicbrainzngs.get_release_group_by_id(release_group_id)
release_group = __musicbrainz_request(lambda: musicbrainzngs.get_release_group_by_id(release_group_id))
if release_group is None:
return year
if 'first-release-date' not in release_group['release-group']:
return year
Generated
+3425
View File
File diff suppressed because it is too large. Load diff