mirror of
https://github.com/rakuri255/UltraSinger.git
synced 2026-11-20 13:07:52 +09:00
Merge branch 'rakuri255:main' into main
This commit is contained in:
46 files changed
+5159
-1287
No files matched your search
@@ -0,0 +1,88 @@
|
||||
name: Build container image
|
||||
on:
|
||||
push:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
GITHUB_REPOSITORY_NAME_PART_SLUG_URL: '' # filled by rlespinasse/github-slug-action@v5
|
||||
REGISTRY_USER: ${{ github.actor }}
|
||||
REGISTRY_PASSWORD: ${{ github.token }}
|
||||
|
||||
jobs:
|
||||
push-ghcr:
|
||||
name: Build and push image
|
||||
runs-on: ubuntu-22.04
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
attestations: write
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Inject enhanced GitHub environment variables
|
||||
uses: rlespinasse/github-slug-action@v5 # https://github.com/rlespinasse/github-slug-action
|
||||
- name: lowercase IMAGE_REGISTRY
|
||||
run: |
|
||||
echo "IMAGE_REGISTRY=ghcr.io/${GITHUB_REPOSITORY_OWNER_PART_SLUG@L}" >> "${GITHUB_ENV}"
|
||||
- name: Clear space
|
||||
run: |
|
||||
df -h /
|
||||
sudo rm -rf \
|
||||
"$AGENT_TOOLSDIRECTORY" \
|
||||
/opt/google/chrome \
|
||||
/opt/microsoft/msedge \
|
||||
/opt/microsoft/powershell \
|
||||
/opt/pipx \
|
||||
/usr/lib/mono \
|
||||
/usr/local/julia* \
|
||||
/usr/local/lib/android \
|
||||
/usr/local/lib/node_modules \
|
||||
/usr/local/share/chromium \
|
||||
/usr/local/share/powershell \
|
||||
/usr/share/dotnet \
|
||||
/usr/share/swift
|
||||
df -h /
|
||||
# Determine tags using Docker Metadata action https://github.com/docker/metadata-action
|
||||
- name: Docker Metadata
|
||||
id: docker-metadata
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
tags: |
|
||||
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
|
||||
type=edge
|
||||
type=sha
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=schedule
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=semver,pattern={{major}}
|
||||
# Build image using Buildah action https://github.com/redhat-actions/buildah-build
|
||||
- name: Build Image
|
||||
id: build_image
|
||||
uses: redhat-actions/buildah-build@v2
|
||||
with:
|
||||
image: ${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG_URL }}
|
||||
tags: ${{ steps.docker-metadata.outputs.tags }}
|
||||
containerfiles: |
|
||||
./Dockerfile
|
||||
oci: true
|
||||
layers: true
|
||||
labels: |
|
||||
${{ steps.docker-metadata.outputs.labels }}
|
||||
# Push the image to GHCR (Image Registry) https://github.com/redhat-actions/push-to-registry
|
||||
- name: Push To Registry
|
||||
uses: redhat-actions/push-to-registry@v2
|
||||
id: push
|
||||
with:
|
||||
image: ${{ steps.build_image.outputs.image }}
|
||||
tags: ${{ steps.build_image.outputs.tags }}
|
||||
registry: ${{ env.IMAGE_REGISTRY }}
|
||||
username: ${{ env.REGISTRY_USER }}
|
||||
password: ${{ env.REGISTRY_PASSWORD }}
|
||||
extra-args: |
|
||||
--disable-content-trust
|
||||
- name: Echo outputs
|
||||
run: |
|
||||
echo "${{ toJSON(steps.push.outputs) }}"
|
||||
@@ -1,28 +0,0 @@
|
||||
name: docker
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'main'
|
||||
|
||||
jobs:
|
||||
docker:
|
||||
environment: docker
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Delete huge unnecessary tools folder
|
||||
run: rm -rf /opt/hostedtoolcache
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
push: true
|
||||
tags: rakuri255/ultrasinger:latest
|
||||
+28
-13
@@ -3,11 +3,13 @@ name: Check Requirements
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "requirements.txt"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
|
||||
pull_request:
|
||||
paths:
|
||||
- "requirements.txt"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@@ -21,22 +23,35 @@ jobs:
|
||||
TARGET: macOS
|
||||
- os: windows-latest
|
||||
TARGET: Windows
|
||||
python-version: [ "3.10" ]
|
||||
python-version: [ "3.12" ]
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v4
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
- name: Install uv
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Install gpu dependencies
|
||||
if: matrix.TARGET != 'macOS'
|
||||
run: pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu117 --index-url https://download.pytorch.org/whl/cu117
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
shell: bash
|
||||
|
||||
- name: Add uv to PATH (Unix)
|
||||
if: matrix.TARGET != 'Windows'
|
||||
run: echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
shell: bash
|
||||
|
||||
- name: Add uv to PATH (Windows)
|
||||
if: matrix.TARGET == 'Windows'
|
||||
run: echo "$env:USERPROFILE\.local\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
shell: pwsh
|
||||
|
||||
- name: Sync dependencies from pyproject.toml
|
||||
run: uv sync
|
||||
shell: bash
|
||||
|
||||
|
||||
|
||||
@@ -14,10 +14,10 @@ jobs:
|
||||
python-version: ["3.10"]
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v3
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
|
||||
@@ -7,22 +7,25 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [ "3.10" ]
|
||||
python-version: [ "3.12" ]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v3
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
- name: Install uv
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
#pip install -e .
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Sync dependencies from pyproject.toml
|
||||
run: uv sync
|
||||
|
||||
- name: Run pytest
|
||||
run: python -m pytest --import-mode=append pytest/
|
||||
run: uv run pytest --import-mode=append pytest/
|
||||
|
||||
+32
-15
@@ -1,27 +1,44 @@
|
||||
FROM nvidia/cuda:12.6.3-runtime-ubuntu22.04
|
||||
FROM nvidia/cuda:12.8.1-runtime-ubuntu22.04
|
||||
|
||||
# note: the python3-pip package contains Python 3.10 on Ubuntu 22.04
|
||||
# Set timezone and configure non-interactive installation
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV TZ=UTC
|
||||
|
||||
# Install Python 3.12 from deadsnakes PPA and build tools
|
||||
RUN apt-get update \
|
||||
&& apt-get install git python3-pip python3.10-venv ffmpeg curl -y \
|
||||
&& apt-get install -y software-properties-common \
|
||||
&& add-apt-repository ppa:deadsnakes/ppa -y \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y git python3.12 python3.12-venv python3.12-dev ffmpeg curl tzdata \
|
||||
build-essential gcc g++ make \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY . /app/UltraSinger
|
||||
# Install uv
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
ENV PATH="/root/.local/bin:$PATH"
|
||||
ENV UV_LINK_MODE=copy
|
||||
|
||||
# copy pyproject.toml first to leverage container image build cache
|
||||
COPY ./pyproject.toml /app/UltraSinger/pyproject.toml
|
||||
# Need to copy some minimal source structure for editable install
|
||||
RUN mkdir -p /app/UltraSinger/src
|
||||
WORKDIR /app/UltraSinger
|
||||
|
||||
# Install dependencies from pyproject.toml directly without venv (container is already isolated)
|
||||
# Using build isolation (without --no-build-isolation) so uv handles all build dependencies automatically
|
||||
RUN uv pip install --system --python 3.12 -e .
|
||||
|
||||
# Install PyTorch with CUDA support (override the CPU version from pyproject.toml)
|
||||
RUN uv pip install --system --python 3.12 torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu128 --reinstall
|
||||
|
||||
# copy sources late to allow for caching of layers which contain all the dependencies
|
||||
COPY . /app/UltraSinger
|
||||
|
||||
|
||||
# no need to run as root
|
||||
RUN chown -R 1000:1000 /app/UltraSinger
|
||||
USER 1000:1000
|
||||
|
||||
# setup venv
|
||||
ENV VIRTUAL_ENV=/app/UltraSinger/.venv
|
||||
RUN python3 -m venv $VIRTUAL_ENV
|
||||
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||
|
||||
# install dependencies
|
||||
RUN pip install --no-cache-dir -r requirements.txt \
|
||||
&& pip install --no-cache-dir torch==2.3.0 torchvision==0.18.0 torchaudio==2.3.0 --index-url https://download.pytorch.org/whl/cu121 \
|
||||
&& pip install --no-cache-dir tensorflow[and-cuda]==2.16.1
|
||||
|
||||
WORKDIR /app/UltraSinger/src
|
||||
CMD ["bash" ]
|
||||
CMD ["bash"]
|
||||
@@ -17,7 +17,7 @@
|
||||
|
||||
> ⚠️ _This project is still under development!_
|
||||
|
||||
UltraSinger is a tool to automatically create UltraStar.txt, midi and notes from music.
|
||||
UltraSinger is a tool to automatically create UltraStar.txt, midi and notes from music.
|
||||
It automatically pitches UltraStar files, adding text and tapping to UltraStar files and creates separate UltraStar karaoke files.
|
||||
It also can re-pitch current UltraStar files and calculates the possible in-game score.
|
||||
|
||||
@@ -69,39 +69,39 @@ This will help me a lot to keep this project alive and improve it.
|
||||
|
||||
### Installation
|
||||
|
||||
* Install Python 3.10 **(older and newer versions has some breaking changes)**. [Download](https://www.python.org/downloads/)
|
||||
* Install Python 3.12 **(older/newer versions may have compatibility issues)**. [Download](https://www.python.org/downloads/)
|
||||
* Also download or install ffmpeg with PATH. [Download](https://www.ffmpeg.org/download.html)
|
||||
* Go to folder `install` and run install script for your OS.
|
||||
* Choose `GPU` if you have an nvidia CUDA GPU.
|
||||
* Choose `CPU` if you don't have an nvidia CUDA GPU.
|
||||
* Go to folder `install` and run install script for your OS:
|
||||
* Choose `GPU` if you have an NVIDIA CUDA GPU.
|
||||
* Choose `CPU` if you don't have an NVIDIA GPU or want CPU-only processing.
|
||||
|
||||
### Run
|
||||
|
||||
* In root folder just run `run_on_windows.bat` or `run_on_linux.sh` to start the app.
|
||||
* Now you can use the UltraSinger source code with `py UltraSinger.py [opt] [mode] [transcription] [pitcher] [extra]`. See [How to use](#how-to-use) for more information.
|
||||
* In root folder just run `run_on_windows.bat`, `run_on_linux.sh` or `run_on_mac.command` to start the app.
|
||||
* Now you can use the UltraSinger source code with `py UltraSinger.py [opt] [mode] [transcription] [pitcher] [extra]`. See [How to use](#-how-to-use-the-app) for more information.
|
||||
|
||||
## 📖 How to use the App
|
||||
|
||||
_Not all options working now!_
|
||||
```commandline
|
||||
UltraSinger.py [opt] [mode] [transcription] [pitcher] [extra]
|
||||
|
||||
|
||||
[opt]
|
||||
-h This help text.
|
||||
-i Ultrastar.txt
|
||||
audio like .mp3, .wav, youtube link
|
||||
audio/video like .mp3, .mp4, .wav, youtube link
|
||||
-o Output folder
|
||||
|
||||
|
||||
[mode]
|
||||
## if INPUT is audio ##
|
||||
default (Full Automatic Mode) - Creates all, depending on command line options
|
||||
--interactive - Interactive Mode. All options are asked at runtime for easier configuration
|
||||
|
||||
|
||||
# Single file creation selection is in progress, you currently getting all!
|
||||
(-u Create ultrastar txt file) # In Progress
|
||||
(-m Create midi file) # In Progress
|
||||
(-s Create sheet file) # In Progress
|
||||
|
||||
|
||||
## if INPUT is ultrastar.txt ##
|
||||
default Creates all
|
||||
|
||||
@@ -117,13 +117,8 @@ _Not all options working now!_
|
||||
--language Override the language detected by whisper, does not affect transcription but steps after transcription
|
||||
--whisper_batch_size Reduce if low on GPU mem >> ((default) is 16)
|
||||
--whisper_compute_type Change to "int8" if low on GPU mem (may reduce accuracy) >> ((default) is "float16" for cuda devices, "int8" for cpu)
|
||||
--keep_numbers Numbers will be transcribed as numerics instead of as words
|
||||
|
||||
[pitcher]
|
||||
# Default is crepe
|
||||
--crepe tiny|full >> ((default) is full)
|
||||
--crepe_step_size unit is miliseconds >> ((default) is 10)
|
||||
|
||||
--keep_numbers Numbers will be transcribed as numerics instead of as words
|
||||
|
||||
[extra]
|
||||
--disable_hyphenation Disable word hyphenation. Hyphenation is enabled by default.
|
||||
--disable_separation Disable track separation. Track separation is enabled by default.
|
||||
@@ -131,18 +126,18 @@ _Not all options working now!_
|
||||
--create_audio_chunks Enable creation of audio chunks. Audio chunks are disabled by default.
|
||||
--keep_cache Keep cache folder after creation. Cache folder is removed by default.
|
||||
--plot Enable creation of plots. Plots are disabled by default.
|
||||
--quantize_to_key Quantize notes to detected musical key. Removes pitch slides and out-of-key notes. >> ((default) is enabled)
|
||||
--format_version 0.3.0|1.0.0|1.1.0|1.2.0 >> ((default) is 1.2.0)
|
||||
--musescore_path path to MuseScore executable
|
||||
--keep_numbers Transcribe numbers as digits and not words
|
||||
--ffmpeg Path to ffmpeg and ffprobe executable
|
||||
|
||||
|
||||
[yt-dlp]
|
||||
--cookiefile File name where cookies should be read from
|
||||
|
||||
[device]
|
||||
--force_cpu Force all steps to be processed on CPU.
|
||||
--force_whisper_cpu Only whisper will be forced to cpu
|
||||
--force_crepe_cpu Only crepe will be forced to cpu
|
||||
```
|
||||
|
||||
For standard use, you only need to use [opt]. All other options are optional.
|
||||
@@ -155,7 +150,7 @@ default (Full Automatic Mode) - Creates all, depending on command line options
|
||||
```commandline
|
||||
--interactive
|
||||
```
|
||||
#### Audio (full automatic)
|
||||
#### Audio / Video (full automatic)
|
||||
|
||||
##### Local file
|
||||
|
||||
@@ -196,8 +191,8 @@ For the first test run, use the `tiny`, to be accurate use the `large-v2` model.
|
||||
|
||||
##### Whisper languages
|
||||
|
||||
Currently provided default language models are `en, fr, de, es, it, ja, zh, nl, uk, pt`.
|
||||
If the language is not in this list, you need to find a phoneme-based ASR model from
|
||||
Currently provided default language models are `en, fr, de, es, it, ja, zh, nl, uk, pt`.
|
||||
If the language is not in this list, you need to find a phoneme-based ASR model from
|
||||
[🤗 huggingface model hub](https://huggingface.co). It will download automatically.
|
||||
|
||||
Example for romanian:
|
||||
@@ -207,9 +202,9 @@ Example for romanian:
|
||||
|
||||
#### ✍️ Hyphenation
|
||||
|
||||
Is on by default. Can also be deactivated if hyphenation does not produce
|
||||
anything useful. Note that the word is simply split,
|
||||
without paying attention to whether the separated word really
|
||||
Is on by default. Can also be deactivated if hyphenation does not produce
|
||||
anything useful. Note that the word is simply split,
|
||||
without paying attention to whether the separated word really
|
||||
starts at the place or is heard. To disable:
|
||||
|
||||
```commandline
|
||||
@@ -218,24 +213,28 @@ starts at the place or is heard. To disable:
|
||||
|
||||
### 👂 Pitcher
|
||||
|
||||
Pitching is done with the `crepe` model.
|
||||
Also consider that a bigger model is more accurate, but also takes longer to pitch.
|
||||
For just testing you should use `tiny`.
|
||||
If you want solid accurate, then use the `full` model.
|
||||
|
||||
```commandline
|
||||
-i XYZ --crepe full
|
||||
```
|
||||
Pitching is done with the `SwiftF0` model, which is faster and more accurate than CREPE.
|
||||
SwiftF0 automatically detects pitch frequencies between 46.875 Hz (G1) and 2093.75 Hz (C7).
|
||||
UltraSinger uses 60hz and 400hz
|
||||
|
||||
### 👄 Separation
|
||||
|
||||
The vocals are separated from the audio before they are passed to the models. If problems occur with this,
|
||||
The vocals are separated from the audio before they are passed to the models. If problems occur with this,
|
||||
you have the option to disable this function; in which case the original audio file is used instead.
|
||||
|
||||
```commandline
|
||||
-i XYZ --disable_separation
|
||||
-i XYZ --disable_separation
|
||||
```
|
||||
|
||||
### 🎵 Key Quantization
|
||||
|
||||
By default, UltraSinger quantizes all detected notes to the detected musical key of the song. This helps to:
|
||||
* Remove pitch slides and vocal transitions between notes
|
||||
* Correct out-of-key notes from pitch detection errors
|
||||
|
||||
If you want to keep the raw pitch detection without quantization, set `quantize_to_key = False`.
|
||||
For most songs quantization should produces better results.
|
||||
|
||||
### Sheet Music
|
||||
|
||||
For Sheet Music generation you need to have `MuseScore` installed on your system.
|
||||
@@ -262,11 +261,11 @@ You can choose between different format versions. The default is `1.2.0`.
|
||||
|
||||
### 🏆 Ultrastar Score Calculation
|
||||
|
||||
The score that the singer in the audio would receive will be measured.
|
||||
You get 2 scores, simple and accurate. You wonder where the difference is?
|
||||
Ultrastar is not interested in pitch hights. As long as it is in the pitch range A-G you get one point.
|
||||
This makes sense for the game, because otherwise men don't get points for high female voices and women don't get points
|
||||
for low male voices. Accurate is the real tone specified in the txt. I had txt files where the pitch was in a range not
|
||||
The score that the singer in the audio would receive will be measured.
|
||||
You get 2 scores, simple and accurate. You wonder where the difference is?
|
||||
Ultrastar is not interested in pitch hights. As long as it is in the pitch range A-G you get one point.
|
||||
This makes sense for the game, because otherwise men don't get points for high female voices and women don't get points
|
||||
for low male voices. Accurate is the real tone specified in the txt. I had txt files where the pitch was in a range not
|
||||
singable by humans, but you could still reach the 10k points in the game. The accuracy is important here, because from
|
||||
this MIDI and sheet are created. And you also want to have accurate files
|
||||
|
||||
@@ -275,34 +274,24 @@ this MIDI and sheet are created. And you also want to have accurate files
|
||||
|
||||
With a GPU you can speed up the process. Also the quality of the transcription and pitching is better.
|
||||
|
||||
You need a cuda device for this to work. Sorry, there is no cuda device for macOS.
|
||||
You need an NVIDIA CUDA device for this to work. Sorry, there is no CUDA device for macOS.
|
||||
|
||||
It is optional (but recommended) to install the cuda driver for your gpu: see [driver](https://developer.nvidia.com/cuda-downloads).
|
||||
Install torch with cuda separately in your `venv`. See [tourch+cuda](https://pytorch.org/get-started/locally/).
|
||||
Also check you GPU cuda support. See [cuda support](https://gist.github.com/standaloneSA/99788f30466516dbcc00338b36ad5acf)
|
||||
For GPU support on Windows and Linux, the installation script automatically installs PyTorch with CUDA support.
|
||||
|
||||
Command for `pip`:
|
||||
It is optional (but recommended) to install the CUDA driver for your GPU: see [CUDA driver](https://developer.nvidia.com/cuda-downloads).
|
||||
Also check your GPU CUDA support. See [CUDA support](https://gist.github.com/standaloneSA/99788f30466516dbcc00338b36ad5acf)
|
||||
|
||||
For manual installation, you can use:
|
||||
```bash
|
||||
uv pip install --index-url https://download.pytorch.org/whl/cu121 torch torchvision torchaudio
|
||||
```
|
||||
pip3 install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2+cu117 --index-url https://download.pytorch.org/whl/cu117
|
||||
```
|
||||
|
||||
When you want to use `conda` instead you need a [different installation command](https://pytorch.org/get-started/locally/).
|
||||
|
||||
#### Considerations for Windows users
|
||||
|
||||
The pitch tracker used by UltraSinger (crepe) uses TensorFlow as its backend.
|
||||
TensorFlow dropped GPU support for Windows for versions >2.10 as you can see in this [release note](https://github.com/tensorflow/tensorflow/releases/tag/v2.11.1) and their [installation instructions](https://www.tensorflow.org/install/pip#windows-native).
|
||||
|
||||
For now UltraSinger runs the latest version available that still supports GPUs on windows.
|
||||
|
||||
For running later versions of TensorFlow on windows while still taking advantage of GPU support the suggested solution is to [run UltraSinger in a container](container/README.md).
|
||||
|
||||
#### Crashes due to low VRAM
|
||||
|
||||
If something crashes because of low VRAM then use a smaller model.
|
||||
If something crashes because of low VRAM then use a smaller Whisper model.
|
||||
Whisper needs more than 8GB VRAM in the `large` model!
|
||||
|
||||
You can also force cpu usage with the extra option `--force_cpu`.
|
||||
You can also force CPU usage with the extra option `--force_cpu`.
|
||||
|
||||
### 📦 Containerized (Docker or Podman)
|
||||
|
||||
|
||||
+9
-2
@@ -1,5 +1,5 @@
|
||||
# Version: 0.0.13dev6
|
||||
Date: 2024.12.26
|
||||
# Version: 0.0.13dev16
|
||||
Date: 2026.03.04
|
||||
- Changes:
|
||||
- Download Cover from MusicBrainz
|
||||
- Fix selection of whisper model
|
||||
@@ -10,6 +10,13 @@ Date: 2024.12.26
|
||||
- Use user defined ffmpeg path
|
||||
- Show GPU Name and VRAM
|
||||
- No linebreak before the last 'E' word for UltraStar txt files
|
||||
- Download yt video/audio only once
|
||||
- Support for video as input
|
||||
- Optimise scale detection
|
||||
- Added quantization by key
|
||||
- Changed installer to uv
|
||||
- Drop crepe for SwiftF0
|
||||
- upgrade to python 3.12
|
||||
|
||||
# Version: 0.0.12
|
||||
Date: 2024.12.19
|
||||
|
||||
+62
-46
@@ -1,48 +1,64 @@
|
||||
{
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": [],
|
||||
"gpuType": "T4"
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
},
|
||||
"accelerator": "GPU"
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": [],
|
||||
"gpuType": "T4"
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "Hki_kX_kSbZX"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%cd /content\n",
|
||||
"!git clone https://github.com/rakuri255/UltraSinger.git\n",
|
||||
"%cd /content/UltraSinger\n",
|
||||
"!pip install --no-cache-dir -r requirements.txt\n",
|
||||
"!pip install --no-cache-dir torch==2.3.0 torchvision==0.18.0 torchaudio==2.3.0 --index-url https://download.pytorch.org/whl/cu121\n",
|
||||
"!pip install --no-cache-dir tensorflow[and-cuda]==2.17.1\n",
|
||||
"%cd /content/UltraSinger/src"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"!python UltraSinger.py -i https://www.youtube.com/watch?v=YwNs1Z0qRY0 -o /content/output"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "O0j4vUW0YAG2"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
}
|
||||
]
|
||||
}
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
},
|
||||
"accelerator": "GPU"
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "Hki_kX_kSbZX"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%cd /content\n",
|
||||
"\n",
|
||||
"# Remove existing directory if present\n",
|
||||
"#!rm -rf UltraSinger\n",
|
||||
"\n",
|
||||
"# Clone specific branch\n",
|
||||
"!git clone https://github.com/rakuri255/UltraSinger.git\n",
|
||||
"%cd /content/UltraSinger\n",
|
||||
"\n",
|
||||
"# Install uv\n",
|
||||
"!curl -LsSf https://astral.sh/uv/install.sh | sh\n",
|
||||
"\n",
|
||||
"# Add uv to PATH and configure matplotlib\n",
|
||||
"import os\n",
|
||||
"os.environ['PATH'] = f\"/root/.local/bin:{os.environ['PATH']}\"\n",
|
||||
"os.environ['UV_LINK_MODE'] = 'copy'\n",
|
||||
"os.environ['MPLBACKEND'] = 'Agg'\n",
|
||||
"\n",
|
||||
"# Sync dependencies from pyproject.toml\n",
|
||||
"!uv sync\n",
|
||||
"\n",
|
||||
"# Install PyTorch with CUDA support\n",
|
||||
"!uv pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu128 --force-reinstall\n",
|
||||
"\n",
|
||||
"%cd /content/UltraSinger/src"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "!../.venv/bin/python UltraSinger.py -i https://www.youtube.com/watch?v=YwNs1Z0qRY0 -o /content/output\n",
|
||||
"metadata": {
|
||||
"id": "O0j4vUW0YAG2"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
}
|
||||
]
|
||||
}
|
||||
+6
-4
@@ -7,9 +7,11 @@
|
||||
```commandline
|
||||
git clone https://github.com/rakuri255/UltraSinger.git
|
||||
```
|
||||
1. build the container image
|
||||
```commandline
|
||||
1. build or pull the container image
|
||||
```bash
|
||||
podman build -t ultrasinger .
|
||||
# or
|
||||
podman pull ghcr.io/rakuri255/ultrasinger:<version> # replace <version> with the desired version or 'latest'
|
||||
```
|
||||
1. run the container (note that the first time you run the container models will be downloaded which may take a while)
|
||||
1. PowerShell:
|
||||
@@ -20,7 +22,7 @@
|
||||
--device nvidia.com/gpu=all `
|
||||
-v $env:USERPROFILE\.cache:/app/UltraSinger/src/.cache `
|
||||
-v <desired-output-folder>:/app/UltraSinger/src/output `
|
||||
ultrasinger `
|
||||
ghcr.io/rakuri255/ultrasinger `
|
||||
python3 UltraSinger.py -i <refer to top-level README.md>
|
||||
|
||||
# explanation:
|
||||
@@ -29,7 +31,7 @@
|
||||
--device nvidia.com/gpu=all ` # optional, enables GPU acceleration if available, requires step 1.ii
|
||||
-v $env:USERPROFILE\.cache:/app/UltraSinger/src/.cache ` # cache directory for models
|
||||
-v <desired-output-folder>:/app/UltraSinger/src/output ` # output directory
|
||||
ultrasinger ` # container image name, we built this in step 3
|
||||
ghcr.io/rakuri255/ultrasinger ` # latest container image, set to 'ultrasinger' to use the locally built image
|
||||
python3 UltraSinger.py -i <refer to top-level README.md> # run UltraSinger, refer to top-level README.md for all options
|
||||
```
|
||||
1. Bash:
|
||||
|
||||
@@ -1,7 +1,35 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
cd ..
|
||||
python3.10 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
cd ../..
|
||||
|
||||
# Set link mode to copy to avoid hardlink warnings
|
||||
export UV_LINK_MODE=copy
|
||||
|
||||
# Install uv if not already installed
|
||||
if ! command -v uv &> /dev/null; then
|
||||
echo "Installing uv..."
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# Update PATH for current session
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
fi
|
||||
|
||||
# Verify uv is available
|
||||
if ! command -v uv &> /dev/null; then
|
||||
echo "Error: uv could not be found or installed"
|
||||
echo "Please ensure your shell PATH includes ~/.local/bin"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "uv version:"
|
||||
uv --version
|
||||
|
||||
echo "Syncing dependencies with uv..."
|
||||
uv sync --extra linux
|
||||
|
||||
echo "Installation completed successfully!"
|
||||
echo "To run UltraSinger:"
|
||||
echo " source .venv/bin/activate"
|
||||
echo " cd src"
|
||||
echo " py UltraSinger.py"
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
cd ../..
|
||||
|
||||
# Set link mode to copy to avoid hardlink warnings
|
||||
export UV_LINK_MODE=copy
|
||||
|
||||
# Install uv if not already installed
|
||||
if ! command -v uv &> /dev/null; then
|
||||
echo "Installing uv..."
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# Update PATH for current session
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
fi
|
||||
|
||||
# Verify uv is available
|
||||
if ! command -v uv &> /dev/null; then
|
||||
echo "Error: uv could not be found or installed"
|
||||
echo "Please ensure your shell PATH includes ~/.local/bin"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "uv version:"
|
||||
uv --version
|
||||
|
||||
echo "Syncing dependencies with uv..."
|
||||
uv sync --extra macos
|
||||
|
||||
echo "Installation completed successfully!"
|
||||
echo "To run UltraSinger:"
|
||||
echo " source .venv/bin/activate"
|
||||
echo " cd src"
|
||||
echo " py UltraSinger.py"
|
||||
@@ -1,9 +1,97 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd ..
|
||||
cd ..
|
||||
py -3.10 -m venv .venv
|
||||
SET VenvPythonPath=%CD%\.venv\Scripts\python.exe
|
||||
call %VenvPythonPath% -m pip install -r requirements.txt
|
||||
call %VenvPythonPath% -m pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1
|
||||
endlocal
|
||||
setlocal enabledelayedexpansion
|
||||
|
||||
:: Set link mode to copy to avoid hardlink warnings on different filesystems
|
||||
set UV_LINK_MODE=copy
|
||||
|
||||
:: Navigate to project root
|
||||
pushd "%~dp0"
|
||||
cd /d ..\..
|
||||
echo Current directory: %cd%
|
||||
|
||||
:: Update PATH to include uv installation directory
|
||||
set "PATH=%USERPROFILE%\.local\bin;!PATH!"
|
||||
|
||||
:: Remove old virtual environment if it exists to force recreation with correct Python version
|
||||
::if exist .venv (
|
||||
:: echo Removing old virtual environment...
|
||||
:: rmdir /s /q .venv
|
||||
::)
|
||||
|
||||
:: First, find Python using to get full path
|
||||
set "PYTHON_EXE="
|
||||
|
||||
for %%V in (3.12) do (
|
||||
py -%%V --version >nul 2>&1
|
||||
if !errorlevel! equ 0 (
|
||||
:: Get the full path to the Python executable
|
||||
for /f "delims=" %%P in ('py -%%V -c "import sys; print(sys.executable)"') do (
|
||||
set "PYTHON_EXE=%%P"
|
||||
)
|
||||
goto :found_python
|
||||
)
|
||||
)
|
||||
|
||||
:: Fallback to direct Python installations
|
||||
for %%P in (python3.12 python3 python) do (
|
||||
where %%P >nul 2>&1
|
||||
if !errorlevel! equ 0 (
|
||||
set "PYTHON_EXE=%%P"
|
||||
goto :found_python
|
||||
)
|
||||
)
|
||||
|
||||
:found_python
|
||||
if "!PYTHON_EXE!"=="" (
|
||||
echo Error: No Python 3.12 installation found
|
||||
echo Please install Python 3.12 from python.org
|
||||
echo Note: Python 3.13 is not yet supported due to dependency constraints
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Using Python: !PYTHON_EXE!
|
||||
"!PYTHON_EXE!" --version
|
||||
|
||||
:: Install uv if not already installed
|
||||
where uv >nul 2>&1
|
||||
if !errorlevel! neq 0 (
|
||||
echo Installing uv...
|
||||
powershell -NoProfile -Command "irm https://astral.sh/uv/install.ps1 | iex"
|
||||
)
|
||||
|
||||
:: Wait a moment for uv to be available
|
||||
timeout /t 2 /nobreak >nul
|
||||
|
||||
:: Verify uv is available
|
||||
where uv >nul 2>&1
|
||||
if !errorlevel! neq 0 (
|
||||
echo Error: uv could not be found or installed
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo uv is ready
|
||||
uv --version
|
||||
|
||||
echo Syncing dependencies with uv...
|
||||
uv sync --extra windows --python "!PYTHON_EXE!"
|
||||
if !errorlevel! neq 0 (
|
||||
echo Error during uv sync
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Installing PyTorch CPU version...
|
||||
:: First remove any existing torch installation to avoid RECORD file issues
|
||||
uv pip uninstall torch torchvision torchaudio -y 2>nul
|
||||
:: Install PyTorch CPU version
|
||||
uv pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision torchaudio
|
||||
if !errorlevel! neq 0 (
|
||||
echo Error during PyTorch installation
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Installation completed successfully!
|
||||
pause
|
||||
@@ -1,7 +1,40 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
cd ..
|
||||
python3.10 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
pip install torch==2.3.1+cu118 torchvision==0.18.1+cu118 torchaudio==2.3.1+cu118 --index-url https://download.pytorch.org/whl/cu118
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
cd ../..
|
||||
|
||||
# Set link mode to copy to avoid hardlink warnings on different filesystems
|
||||
export UV_LINK_MODE=copy
|
||||
|
||||
# Install uv if not already installed
|
||||
if ! command -v uv &> /dev/null; then
|
||||
echo "Installing uv..."
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# Update PATH for current session
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
fi
|
||||
|
||||
# Verify uv is available
|
||||
if ! command -v uv &> /dev/null; then
|
||||
echo "Error: uv could not be found or installed"
|
||||
echo "Please ensure your shell PATH includes ~/.local/bin"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "uv version:"
|
||||
uv --version
|
||||
|
||||
# Sync dependencies from pyproject.toml
|
||||
echo "Syncing dependencies from pyproject.toml..."
|
||||
uv sync --extra linux
|
||||
|
||||
# Install PyTorch with CUDA support
|
||||
echo "Installing PyTorch with CUDA support..."
|
||||
uv pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu128 --force-reinstall
|
||||
|
||||
echo "Installation completed successfully!"
|
||||
echo "To run UltraSinger:"
|
||||
echo " source .venv/bin/activate"
|
||||
echo " cd src"
|
||||
echo " python UltraSinger.py"
|
||||
@@ -1,9 +1,98 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd ..
|
||||
cd ..
|
||||
py -3.10 -m venv .venv
|
||||
SET VenvPythonPath=%CD%\.venv\Scripts\python.exe
|
||||
call %VenvPythonPath% -m pip install -r requirements.txt
|
||||
call %VenvPythonPath% -m pip install torch==2.3.1+cu118 torchvision==0.18.1+cu118 torchaudio==2.3.1+cu118 --index-url https://download.pytorch.org/whl/cu118
|
||||
endlocal
|
||||
setlocal enabledelayedexpansion
|
||||
|
||||
:: Set link mode to copy to avoid hardlink warnings on different filesystems
|
||||
set UV_LINK_MODE=copy
|
||||
|
||||
:: Navigate to project root
|
||||
pushd "%~dp0"
|
||||
cd /d ..\..
|
||||
echo Current directory: %cd%
|
||||
|
||||
:: Update PATH to include uv installation directory
|
||||
set "PATH=%USERPROFILE%\.local\bin;!PATH!"
|
||||
|
||||
:: Remove old virtual environment if it exists to force recreation with correct Python version
|
||||
::if exist .venv (
|
||||
:: echo Removing old virtual environment...
|
||||
:: rmdir /s /q .venv
|
||||
::)
|
||||
|
||||
:: First, find Python using to get full path
|
||||
set "PYTHON_EXE="
|
||||
|
||||
for %%V in (3.12) do (
|
||||
py -%%V --version >nul 2>&1
|
||||
if !errorlevel! equ 0 (
|
||||
:: Get the full path to the Python executable
|
||||
for /f "delims=" %%P in ('py -%%V -c "import sys; print(sys.executable)"') do (
|
||||
set "PYTHON_EXE=%%P"
|
||||
)
|
||||
goto :found_python
|
||||
)
|
||||
)
|
||||
|
||||
:: Fallback to direct Python installations
|
||||
for %%P in (python3.12 python3 python) do (
|
||||
where %%P >nul 2>&1
|
||||
if !errorlevel! equ 0 (
|
||||
set "PYTHON_EXE=%%P"
|
||||
goto :found_python
|
||||
)
|
||||
)
|
||||
|
||||
:found_python
|
||||
if "!PYTHON_EXE!"=="" (
|
||||
echo Error: No Python 3.12 installation found
|
||||
echo Please install Python 3.12 from python.org
|
||||
echo Note: Python 3.13 is not yet supported due to dependency constraints
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Using Python: !PYTHON_EXE!
|
||||
"!PYTHON_EXE!" --version
|
||||
|
||||
:: Install uv if not already installed
|
||||
where uv >nul 2>&1
|
||||
if !errorlevel! neq 0 (
|
||||
echo Installing uv...
|
||||
powershell -NoProfile -Command "irm https://astral.sh/uv/install.ps1 | iex"
|
||||
)
|
||||
|
||||
:: Wait a moment for uv to be available
|
||||
timeout /t 2 /nobreak >nul
|
||||
|
||||
:: Verify uv is available
|
||||
where uv >nul 2>&1
|
||||
if !errorlevel! neq 0 (
|
||||
echo Error: uv could not be found or installed
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo uv is ready
|
||||
uv --version
|
||||
|
||||
echo Syncing dependencies with uv...
|
||||
uv sync --extra windows --python "!PYTHON_EXE!"
|
||||
if !errorlevel! neq 0 (
|
||||
echo Error during uv sync
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Installing PyTorch with CUDA support (as recommended by WhisperX)...
|
||||
:: First remove any existing torch installation to avoid RECORD file issues
|
||||
uv pip uninstall torch torchvision torchaudio -y 2>nul
|
||||
:: Install PyTorch 2.8.0 with CUDA for WhisperX compatibility
|
||||
:: This version includes cuDNN 9.x which is compatible with the latest PyTorch
|
||||
uv pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu128 --force-reinstall
|
||||
if !errorlevel! neq 0 (
|
||||
echo Error during PyTorch installation
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Installation completed successfully!
|
||||
pause
|
||||
@@ -0,0 +1,100 @@
|
||||
@echo off
|
||||
echo ========================================
|
||||
echo CUDA 11.x Libraries Fix for WhisperX/Pyannote
|
||||
echo ========================================
|
||||
echo.
|
||||
echo This script will download and install CUDA 11.x DLLs
|
||||
echo that are required by pyannote.audio (used by WhisperX)
|
||||
echo.
|
||||
echo PyTorch 2.8.0 includes CUDA 12.x, but pyannote.audio
|
||||
echo still requires CUDA 11.x DLLs to function properly.
|
||||
echo.
|
||||
|
||||
cd /d "%~dp0\.."
|
||||
|
||||
echo Checking for virtual environment...
|
||||
if not exist ".venv\Scripts\activate.bat" (
|
||||
echo Error: Virtual environment not found at .venv
|
||||
echo Please create the virtual environment first.
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Activating virtual environment...
|
||||
call .venv\Scripts\activate.bat
|
||||
|
||||
if %errorlevel% neq 0 (
|
||||
echo Error activating virtual environment
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Creating temporary directory...
|
||||
if not exist temp_cuda11 mkdir temp_cuda11
|
||||
cd temp_cuda11
|
||||
|
||||
echo.
|
||||
echo Downloading CUDA 11.8 Toolkit Libraries (Windows)...
|
||||
echo This may take a few minutes...
|
||||
echo.
|
||||
echo Downloading cuBLAS 11.x...
|
||||
powershell -Command "& {[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; Invoke-WebRequest -Uri 'https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-11.11.3.6-archive.zip' -OutFile 'cublas.zip'}"
|
||||
|
||||
if %errorlevel% neq 0 (
|
||||
echo Error downloading cuBLAS 11.x
|
||||
echo.
|
||||
echo Please check your internet connection or download manually from:
|
||||
echo https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/
|
||||
echo.
|
||||
cd ..
|
||||
deactivate
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo.
|
||||
echo Extracting CUDA 11.x Libraries...
|
||||
if exist cublas.zip (
|
||||
powershell -Command "Expand-Archive -Path 'cublas.zip' -DestinationPath '.' -Force"
|
||||
)
|
||||
|
||||
echo.
|
||||
echo Copying CUDA 11.x DLLs to PyTorch installation...
|
||||
set "COPIED=0"
|
||||
|
||||
if exist "libcublas-windows-x86_64-11.11.3.6-archive\bin\*.dll" (
|
||||
copy /Y "libcublas-windows-x86_64-11.11.3.6-archive\bin\*.dll" "%VIRTUAL_ENV%\Lib\site-packages\torch\lib\"
|
||||
set "COPIED=1"
|
||||
)
|
||||
|
||||
if "%COPIED%"=="0" (
|
||||
echo Error: Could not find CUDA 11.x DLLs in extracted files
|
||||
echo Expected path: libcublas-windows-x86_64-11.11.3.6-archive\bin\
|
||||
cd ..
|
||||
deactivate
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo.
|
||||
echo Successfully copied CUDA 11.x DLLs!
|
||||
echo Listing copied cuBLAS files:
|
||||
dir "%VIRTUAL_ENV%\Lib\site-packages\torch\lib\cublas*.dll" /b 2>nul
|
||||
|
||||
echo.
|
||||
echo Cleaning up temporary files...
|
||||
cd ..
|
||||
rmdir /s /q temp_cuda11
|
||||
|
||||
echo Deactivating virtual environment...
|
||||
deactivate
|
||||
|
||||
echo.
|
||||
echo ========================================
|
||||
echo CUDA 11.x Fix completed successfully!
|
||||
echo ========================================
|
||||
echo.
|
||||
echo You can now run UltraSinger without CUDA 11.x DLL errors.
|
||||
echo.
|
||||
pause
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
@echo off
|
||||
echo ========================================
|
||||
echo cuDNN 8.x Fix for WhisperX/Pyannote
|
||||
echo ========================================
|
||||
echo.
|
||||
echo This script will download and install cuDNN 8.x DLLs
|
||||
echo that are required by pyannote.audio (used by WhisperX)
|
||||
echo.
|
||||
echo PyTorch 2.8.0 includes cuDNN 9.x, but pyannote.audio
|
||||
echo still requires cuDNN 8.x DLLs to function properly.
|
||||
echo.
|
||||
|
||||
cd /d "%~dp0\.."
|
||||
|
||||
echo Checking for virtual environment...
|
||||
if not exist ".venv\Scripts\activate.bat" (
|
||||
echo Error: Virtual environment not found at .venv
|
||||
echo Please create the virtual environment first.
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Activating virtual environment...
|
||||
call .venv\Scripts\activate.bat
|
||||
|
||||
if %errorlevel% neq 0 (
|
||||
echo Error activating virtual environment
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Creating temporary directory...
|
||||
if not exist temp_cudnn mkdir temp_cudnn
|
||||
cd temp_cudnn
|
||||
|
||||
echo.
|
||||
echo Downloading cuDNN 8.9.7 for CUDA 11.x (Windows)...
|
||||
echo This may take a few minutes...
|
||||
powershell -Command "& {[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; Invoke-WebRequest -Uri 'https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/windows-x86_64/cudnn-windows-x86_64-8.9.7.29_cuda11-archive.zip' -OutFile 'cudnn.zip'}"
|
||||
|
||||
if %errorlevel% neq 0 (
|
||||
echo Error downloading cuDNN 8.9.7
|
||||
echo.
|
||||
echo Alternative: Trying to extract from installed torch packages...
|
||||
echo Searching for existing PyTorch installations with cuDNN 8.x...
|
||||
|
||||
REM Try to find cuDNN 8 DLLs in pip cache or other Python installations
|
||||
powershell -Command "& {$found = $false; Get-ChildItem -Path $env:LOCALAPPDATA\pip\cache -Recurse -Filter 'cudnn64_8.dll' -ErrorAction SilentlyContinue | ForEach-Object {echo $_.FullName; $found = $true}; if (-not $found) {exit 1}}"
|
||||
|
||||
if %errorlevel% neq 0 (
|
||||
echo Error: Could not find or download cuDNN 8.x DLLs
|
||||
echo.
|
||||
echo Please manually download cuDNN 8.9.7 from NVIDIA:
|
||||
echo https://developer.nvidia.com/rdp/cudnn-archive
|
||||
echo.
|
||||
cd ..
|
||||
deactivate
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
)
|
||||
|
||||
echo.
|
||||
echo Extracting cuDNN 8.x DLLs...
|
||||
if exist cudnn.zip (
|
||||
powershell -Command "Expand-Archive -Path 'cudnn.zip' -DestinationPath '.' -Force"
|
||||
)
|
||||
|
||||
echo.
|
||||
echo Copying cuDNN 8.x DLLs to PyTorch installation...
|
||||
if exist "cudnn-windows-x86_64-8.9.7.29_cuda11-archive\bin\cudnn*.dll" (
|
||||
copy /Y "cudnn-windows-x86_64-8.9.7.29_cuda11-archive\bin\cudnn*.dll" "%VIRTUAL_ENV%\Lib\site-packages\torch\lib\"
|
||||
echo.
|
||||
echo Successfully copied cuDNN 8.x DLLs!
|
||||
echo Listing copied files:
|
||||
dir "%VIRTUAL_ENV%\Lib\site-packages\torch\lib\cudnn*.dll" /b
|
||||
) else (
|
||||
echo Error: Could not find cuDNN DLLs in extracted files
|
||||
echo Expected path: cudnn-windows-x86_64-8.9.7.29_cuda11-archive\bin\
|
||||
cd ..
|
||||
deactivate
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo.
|
||||
echo Cleaning up temporary files...
|
||||
cd ..
|
||||
rmdir /s /q temp_cudnn
|
||||
|
||||
echo Deactivating virtual environment...
|
||||
deactivate
|
||||
|
||||
echo.
|
||||
echo ========================================
|
||||
echo cuDNN 8.x Fix completed successfully!
|
||||
echo ========================================
|
||||
echo.
|
||||
echo You can now run UltraSinger without cuDNN errors.
|
||||
echo.
|
||||
pause
|
||||
|
||||
+47
-10
@@ -1,22 +1,59 @@
|
||||
[build-system]
|
||||
requires = ["setuptools", "setuptools-scm"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
requires = ["hatchling", "setuptools", "wheel", "setuptools_scm"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
requires-python = "3.10"
|
||||
name = "UltraSinger"
|
||||
dynamic = ["dependencies", "optional-dependencies"]
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
dependencies = { file = ["requirements.in"] }
|
||||
optional-dependencies.test = { file = ["requirements-test.txt"] }
|
||||
version = "0.0.13.dev16"
|
||||
description = "A tool to create UltraStar karaoke files from audio files"
|
||||
requires-python = ">=3.12,<3.14"
|
||||
dependencies = [
|
||||
"setuptools",
|
||||
"scipy",
|
||||
"whisperx",
|
||||
"num2words",
|
||||
"inputimeout",
|
||||
"langcodes",
|
||||
"language_data",
|
||||
"packaging",
|
||||
"librosa",
|
||||
"swift-f0",
|
||||
"pydub",
|
||||
"demucs",
|
||||
"ffmpeg-python",
|
||||
"matplotlib",
|
||||
"musicbrainzngs",
|
||||
"python-Levenshtein",
|
||||
"pretty-midi",
|
||||
"unidecode",
|
||||
"pyhyphen",
|
||||
"tqdm",
|
||||
"yt-dlp[default]",
|
||||
"deno",
|
||||
"music21",
|
||||
"dataclasses",
|
||||
"dataclasses-json",
|
||||
"torch==2.8.0",
|
||||
"torchaudio==2.8.0",
|
||||
"torchvision==0.23.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = ["pytest"]
|
||||
windows = []
|
||||
linux = []
|
||||
macos = []
|
||||
dev = ["pytest", "isort", "black", "pylint"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["pytest", "isort", "black", "pylint"]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src"]
|
||||
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath = [
|
||||
"src",
|
||||
]
|
||||
]
|
||||
@@ -18,7 +18,7 @@ class PitcherTest(unittest.TestCase):
|
||||
test_output = root_dir + "/test_output"
|
||||
|
||||
# Act
|
||||
pitched_data = test_subject.get_pitch_with_crepe_file(test_file_abs_path, 'full', device="cuda")
|
||||
pitched_data = test_subject.get_pitch_with_file(test_file_abs_path)
|
||||
# test_subject.get_pitch_with_crepe_file(test_file_abs_path, 'full', 'cpu', batch_size=1024)
|
||||
plot(pitched_data, test_output, title="pitching test")
|
||||
print("done")
|
||||
|
||||
@@ -29,7 +29,8 @@ class TestUltraStarParser(unittest.TestCase):
|
||||
self.assertEqual(result, ("Test Artist - Test Title",
|
||||
os.path.join("path", "to", "output", "Test Artist - Test Title"),
|
||||
os.path.join("path", "to", "input", "test.mp3"),
|
||||
mock_parse.return_value))
|
||||
mock_parse.return_value,
|
||||
"mp3"))
|
||||
#
|
||||
mock_parse.assert_called_once()
|
||||
mock_dirname.assert_called_once()
|
||||
|
||||
@@ -104,26 +104,26 @@ class TestCreateUltrastarTxt(unittest.TestCase):
|
||||
def default_values(default_ultrastar_class, ver):
|
||||
expected_calls = []
|
||||
if version.parse(ver) >= version.parse("1.0.0"):
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VERSION}:{default_ultrastar_class.version}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VERSION.value}:{default_ultrastar_class.version}\n")
|
||||
expected_calls += [
|
||||
f"#{UltrastarTxtTag.ARTIST}:{default_ultrastar_class.artist}\n",
|
||||
f"#{UltrastarTxtTag.TITLE}:{default_ultrastar_class.title}\n",
|
||||
f"#{UltrastarTxtTag.MP3}:{default_ultrastar_class.mp3}\n"
|
||||
f"#{UltrastarTxtTag.ARTIST.value}:{default_ultrastar_class.artist}\n",
|
||||
f"#{UltrastarTxtTag.TITLE.value}:{default_ultrastar_class.title}\n",
|
||||
f"#{UltrastarTxtTag.MP3.value}:{default_ultrastar_class.mp3}\n"
|
||||
]
|
||||
if version.parse(ver) >= version.parse("1.1.0"):
|
||||
expected_calls += [f"#{UltrastarTxtTag.AUDIO}:{default_ultrastar_class.audio}\n"]
|
||||
expected_calls += [f"#{UltrastarTxtTag.AUDIO.value}:{default_ultrastar_class.audio}\n"]
|
||||
if default_ultrastar_class.video is not None:
|
||||
expected_calls += [
|
||||
f"#{UltrastarTxtTag.VIDEO}:{default_ultrastar_class.video}\n"
|
||||
f"#{UltrastarTxtTag.VIDEO.value}:{default_ultrastar_class.video}\n"
|
||||
]
|
||||
if version.parse(ver) >= version.parse("1.2.0"):
|
||||
if default_ultrastar_class.videoUrl is not None:
|
||||
expected_calls += [f"#{UltrastarTxtTag.VIDEOURL}:{default_ultrastar_class.videoUrl}\n"]
|
||||
expected_calls += [f"#{UltrastarTxtTag.VIDEOURL.value}:{default_ultrastar_class.videoUrl}\n"]
|
||||
expected_calls += [
|
||||
f"#{UltrastarTxtTag.BPM}:390.0\n",
|
||||
f"#{UltrastarTxtTag.GAP}:500\n",
|
||||
f"#{UltrastarTxtTag.CREATOR}:{default_ultrastar_class.creator}\n",
|
||||
f"#{UltrastarTxtTag.COMMENT}:{default_ultrastar_class.comment}\n",
|
||||
f"#{UltrastarTxtTag.BPM.value}:390.0\n",
|
||||
f"#{UltrastarTxtTag.GAP.value}:500\n",
|
||||
f"#{UltrastarTxtTag.CREATOR.value}:{default_ultrastar_class.creator}\n",
|
||||
f"#{UltrastarTxtTag.COMMENT.value}:{default_ultrastar_class.comment}\n",
|
||||
": 0 52 1 UltraSinger \n",
|
||||
": 65 39 2 is \n",
|
||||
": 130 52 3 cool! \n",
|
||||
@@ -136,26 +136,27 @@ class TestCreateUltrastarTxt(unittest.TestCase):
|
||||
def full_values(default_ultrastar_class, ver):
|
||||
expected_calls = []
|
||||
if version.parse(ver) >= version.parse("1.0.0"):
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VERSION}:{default_ultrastar_class.version}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.ARTIST}:{default_ultrastar_class.artist}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.TITLE}:{default_ultrastar_class.title}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.YEAR}:{default_ultrastar_class.year}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.LANGUAGE}:German\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.GENRE}:{default_ultrastar_class.genre}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.COVER}:{default_ultrastar_class.cover}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.MP3}:{default_ultrastar_class.mp3}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VERSION.value}:{default_ultrastar_class.version}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.ARTIST.value}:{default_ultrastar_class.artist}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.TITLE.value}:{default_ultrastar_class.title}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.YEAR.value}:{default_ultrastar_class.year}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.LANGUAGE.value}:German\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.GENRE.value}:{default_ultrastar_class.genre}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.COVER.value}:{default_ultrastar_class.cover}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.MP3.value}:{default_ultrastar_class.mp3}\n")
|
||||
if version.parse(ver) >= version.parse("1.1.0"):
|
||||
expected_calls.append(f"#{UltrastarTxtTag.AUDIO}:{default_ultrastar_class.audio}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VOCALS}:{default_ultrastar_class.vocals}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.INSTRUMENTAL}:{default_ultrastar_class.instrumental}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.TAGS}:{default_ultrastar_class.tags}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VIDEO}:{default_ultrastar_class.video}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.AUDIO.value}:{default_ultrastar_class.audio}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VOCALS.value}:{default_ultrastar_class.vocals}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.INSTRUMENTAL.value}:{default_ultrastar_class.instrumental}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VIDEO.value}:{default_ultrastar_class.video}\n")
|
||||
if version.parse(ver) >= version.parse("1.2.0"):
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VIDEOURL}:{default_ultrastar_class.videoUrl}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.BPM}:390.0\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.GAP}:500\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.CREATOR}:{default_ultrastar_class.creator}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.COMMENT}:{default_ultrastar_class.comment}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.VIDEOURL.value}:{default_ultrastar_class.videoUrl}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.BPM.value}:390.0\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.GAP.value}:500\n")
|
||||
if version.parse(ver) >= version.parse("1.1.0"):
|
||||
expected_calls.append(f"#{UltrastarTxtTag.TAGS.value}:{default_ultrastar_class.tags}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.CREATOR.value}:{default_ultrastar_class.creator}\n")
|
||||
expected_calls.append(f"#{UltrastarTxtTag.COMMENT.value}:{default_ultrastar_class.comment}\n")
|
||||
expected_calls.append(": 0 52 1 UltraSinger \n")
|
||||
expected_calls.append(": 65 39 2 is \n")
|
||||
expected_calls.append(": 130 52 3 cool! \n")
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
# requirements.in
|
||||
|
||||
# Pitching
|
||||
crepe
|
||||
# machine learning platform (used for crepe)
|
||||
tensorflow==2.10
|
||||
|
||||
# Audio to wavefile (used for crepe)
|
||||
scipy
|
||||
|
||||
# Transcription
|
||||
#whisperx
|
||||
git+https://github.com/m-bain/whisperx.git
|
||||
# Convert numbers to words in multilanguage (Used because of whisper)
|
||||
num2words
|
||||
# User imput timeout (used in whisper)
|
||||
inputimeout
|
||||
|
||||
# langcodes for creates the language name from the language code. Used for Ultrastar txt
|
||||
langcodes
|
||||
# Version handling (Used in UltraStar txt)
|
||||
packaging
|
||||
|
||||
# Audio analysis
|
||||
librosa~=0.9.2
|
||||
# Audio manipulation
|
||||
pydub
|
||||
# Audio Separation
|
||||
demucs
|
||||
# Ffmpeg wrapper
|
||||
ffmpeg_python
|
||||
|
||||
# Plot engine
|
||||
matplotlib
|
||||
|
||||
# For getting additional song info like cover, genre, year
|
||||
musicbrainzngs
|
||||
# For distance calculation (used in musicbrainz_client)
|
||||
python_Levenshtein
|
||||
|
||||
# For midi file handling
|
||||
pretty_midi
|
||||
# Convert Unicode to ascii (used for midi)
|
||||
unidecode
|
||||
|
||||
# For text hyphenation
|
||||
#PyHyphen
|
||||
PyHyphen-Reloaded
|
||||
|
||||
# Progress bar
|
||||
tqdm
|
||||
|
||||
# yt
|
||||
yt_dlp
|
||||
|
||||
# external cleanup tools
|
||||
## Sort imports
|
||||
isort
|
||||
## Code formatter
|
||||
black
|
||||
## Code analyser
|
||||
pylint
|
||||
|
||||
# For Tests
|
||||
pytest
|
||||
|
||||
# Musical Analysis and Computational Musicology (Used for sheet)
|
||||
music21
|
||||
|
||||
# Data structure
|
||||
dataclasses
|
||||
dataclasses_json
|
||||
|
||||
# Array support
|
||||
numpy<2
|
||||
@@ -1,655 +0,0 @@
|
||||
#
|
||||
# This file is autogenerated by pip-compile with Python 3.10
|
||||
# by the following command:
|
||||
#
|
||||
# pip-compile
|
||||
#
|
||||
absl-py==2.1.0
|
||||
# via
|
||||
# tensorboard
|
||||
# tensorflow
|
||||
aiohttp==3.9.5
|
||||
# via fsspec
|
||||
aiosignal==1.3.1
|
||||
# via aiohttp
|
||||
alembic==1.13.2
|
||||
# via optuna
|
||||
antlr4-python3-runtime==4.9.3
|
||||
# via omegaconf
|
||||
appdirs==1.4.4
|
||||
# via pyhyphen-reloaded
|
||||
asteroid-filterbanks==0.4.0
|
||||
# via pyannote-audio
|
||||
astroid==3.2.4
|
||||
# via pylint
|
||||
astunparse==1.6.3
|
||||
# via tensorflow
|
||||
async-timeout==4.0.3
|
||||
# via aiohttp
|
||||
attrs==23.2.0
|
||||
# via aiohttp
|
||||
audioread==3.0.1
|
||||
# via librosa
|
||||
av==11.0.0
|
||||
# via faster-whisper
|
||||
black==24.4.2
|
||||
# via -r requirements.in
|
||||
cachetools==5.4.0
|
||||
# via google-auth
|
||||
certifi==2024.7.4
|
||||
# via requests
|
||||
cffi==1.16.0
|
||||
# via soundfile
|
||||
chardet==5.2.0
|
||||
# via music21
|
||||
charset-normalizer==3.3.2
|
||||
# via requests
|
||||
click==8.1.7
|
||||
# via
|
||||
# black
|
||||
# nltk
|
||||
# typer
|
||||
cloudpickle==3.0.0
|
||||
# via submitit
|
||||
colorama==0.4.6
|
||||
# via
|
||||
# click
|
||||
# colorlog
|
||||
# pylint
|
||||
# pytest
|
||||
# tqdm
|
||||
coloredlogs==15.0.1
|
||||
# via onnxruntime
|
||||
colorlog==6.8.2
|
||||
# via optuna
|
||||
contourpy==1.2.1
|
||||
# via matplotlib
|
||||
crepe==0.0.16
|
||||
# via -r requirements.in
|
||||
ctranslate2==4.4.0
|
||||
# via
|
||||
# faster-whisper
|
||||
# whisperx
|
||||
cycler==0.12.1
|
||||
# via matplotlib
|
||||
dataclasses==0.6
|
||||
# via -r requirements.in
|
||||
dataclasses-json==0.6.7
|
||||
# via -r requirements.in
|
||||
decorator==5.1.1
|
||||
# via librosa
|
||||
demucs==4.0.1
|
||||
# via -r requirements.in
|
||||
dill==0.3.8
|
||||
# via pylint
|
||||
docopt==0.6.2
|
||||
# via
|
||||
# num2words
|
||||
# pyannote-metrics
|
||||
# pyannote-pipeline
|
||||
dora-search==0.1.12
|
||||
# via demucs
|
||||
einops==0.8.0
|
||||
# via
|
||||
# demucs
|
||||
# pyannote-audio
|
||||
exceptiongroup==1.2.2
|
||||
# via pytest
|
||||
faster-whisper==1.1.0
|
||||
# via whisperx
|
||||
ffmpeg-python==0.2.0
|
||||
# via -r requirements.in
|
||||
filelock==3.15.4
|
||||
# via
|
||||
# huggingface-hub
|
||||
# pyannote-pipeline
|
||||
# torch
|
||||
# transformers
|
||||
flatbuffers==24.3.25
|
||||
# via
|
||||
# onnxruntime
|
||||
# tensorflow
|
||||
fonttools==4.53.1
|
||||
# via matplotlib
|
||||
frozenlist==1.4.1
|
||||
# via
|
||||
# aiohttp
|
||||
# aiosignal
|
||||
fsspec[http]==2024.6.1
|
||||
# via
|
||||
# huggingface-hub
|
||||
# lightning
|
||||
# pytorch-lightning
|
||||
# torch
|
||||
future==1.0.0
|
||||
# via ffmpeg-python
|
||||
gast==0.4.0
|
||||
# via tensorflow
|
||||
google-auth==2.32.0
|
||||
# via
|
||||
# google-auth-oauthlib
|
||||
# tensorboard
|
||||
google-auth-oauthlib==0.4.6
|
||||
# via tensorboard
|
||||
google-pasta==0.2.0
|
||||
# via tensorflow
|
||||
greenlet==3.0.3
|
||||
# via sqlalchemy
|
||||
grpcio==1.65.1
|
||||
# via
|
||||
# tensorboard
|
||||
# tensorflow
|
||||
h5py==3.11.0
|
||||
# via
|
||||
# crepe
|
||||
# tensorflow
|
||||
hmmlearn==0.3.2
|
||||
# via crepe
|
||||
huggingface-hub==0.24.2
|
||||
# via
|
||||
# faster-whisper
|
||||
# pyannote-audio
|
||||
# speechbrain
|
||||
# tokenizers
|
||||
# transformers
|
||||
humanfriendly==10.0
|
||||
# via coloredlogs
|
||||
hyperpyyaml==1.2.2
|
||||
# via speechbrain
|
||||
idna==3.7
|
||||
# via
|
||||
# requests
|
||||
# yarl
|
||||
imageio==2.34.2
|
||||
# via crepe
|
||||
iniconfig==2.0.0
|
||||
# via pytest
|
||||
inputimeout==1.0.4
|
||||
# via -r requirements.in
|
||||
isort==5.13.2
|
||||
# via
|
||||
# -r requirements.in
|
||||
# pylint
|
||||
jinja2==3.1.4
|
||||
# via torch
|
||||
joblib==1.4.2
|
||||
# via
|
||||
# librosa
|
||||
# music21
|
||||
# nltk
|
||||
# scikit-learn
|
||||
# speechbrain
|
||||
jsonpickle==3.2.2
|
||||
# via music21
|
||||
julius==0.2.7
|
||||
# via
|
||||
# demucs
|
||||
# torch-audiomentations
|
||||
keras==2.10.0
|
||||
# via tensorflow
|
||||
keras-preprocessing==1.1.2
|
||||
# via tensorflow
|
||||
kiwisolver==1.4.5
|
||||
# via matplotlib
|
||||
lameenc==1.7.0
|
||||
# via demucs
|
||||
langcodes==3.5.0
|
||||
# via -r requirements.in
|
||||
language-data==1.2.0
|
||||
# via langcodes
|
||||
levenshtein==0.25.1
|
||||
# via python-levenshtein
|
||||
libclang==18.1.1
|
||||
# via tensorflow
|
||||
librosa==0.9.2
|
||||
# via
|
||||
# -r requirements.in
|
||||
# torch-audiomentations
|
||||
lightning==2.3.3
|
||||
# via pyannote-audio
|
||||
lightning-utilities==0.11.6
|
||||
# via
|
||||
# lightning
|
||||
# pytorch-lightning
|
||||
# torchmetrics
|
||||
llvmlite==0.43.0
|
||||
# via numba
|
||||
mako==1.3.5
|
||||
# via alembic
|
||||
marisa-trie==1.2.0
|
||||
# via language-data
|
||||
markdown==3.6
|
||||
# via tensorboard
|
||||
markdown-it-py==3.0.0
|
||||
# via rich
|
||||
markupsafe==2.1.5
|
||||
# via
|
||||
# jinja2
|
||||
# mako
|
||||
# werkzeug
|
||||
marshmallow==3.21.3
|
||||
# via dataclasses-json
|
||||
matplotlib==3.10.0
|
||||
# via
|
||||
# -r requirements.in
|
||||
# crepe
|
||||
# music21
|
||||
# pyannote-metrics
|
||||
mccabe==0.7.0
|
||||
# via pylint
|
||||
mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
mido==1.3.2
|
||||
# via pretty-midi
|
||||
more-itertools==10.3.0
|
||||
# via music21
|
||||
mpmath==1.3.0
|
||||
# via sympy
|
||||
multidict==6.0.5
|
||||
# via
|
||||
# aiohttp
|
||||
# yarl
|
||||
music21==9.3.0
|
||||
# via -r requirements.in
|
||||
musicbrainzngs==0.7.1
|
||||
# via -r requirements.in
|
||||
mypy-extensions==1.0.0
|
||||
# via
|
||||
# black
|
||||
# typing-inspect
|
||||
networkx==3.3
|
||||
# via torch
|
||||
nltk==3.8.1
|
||||
# via whisperx
|
||||
num2words==0.5.14
|
||||
# via -r requirements.in
|
||||
numba==0.60.0
|
||||
# via
|
||||
# librosa
|
||||
# resampy
|
||||
numpy==1.26.4
|
||||
# via
|
||||
# -r requirements.in
|
||||
# asteroid-filterbanks
|
||||
# contourpy
|
||||
# crepe
|
||||
# ctranslate2
|
||||
# h5py
|
||||
# hmmlearn
|
||||
# imageio
|
||||
# keras-preprocessing
|
||||
# librosa
|
||||
# lightning
|
||||
# matplotlib
|
||||
# music21
|
||||
# numba
|
||||
# onnxruntime
|
||||
# openunmix
|
||||
# opt-einsum
|
||||
# optuna
|
||||
# pandas
|
||||
# pretty-midi
|
||||
# pyannote-core
|
||||
# pyannote-metrics
|
||||
# pytorch-lightning
|
||||
# pytorch-metric-learning
|
||||
# resampy
|
||||
# scikit-learn
|
||||
# scipy
|
||||
# speechbrain
|
||||
# tensorboard
|
||||
# tensorboardx
|
||||
# tensorflow
|
||||
# torchmetrics
|
||||
# transformers
|
||||
oauthlib==3.2.2
|
||||
# via requests-oauthlib
|
||||
omegaconf==2.3.0
|
||||
# via
|
||||
# dora-search
|
||||
# pyannote-audio
|
||||
onnxruntime==1.18.1
|
||||
# via faster-whisper
|
||||
openunmix==1.3.0
|
||||
# via demucs
|
||||
opt-einsum==3.3.0
|
||||
# via tensorflow
|
||||
optuna==3.6.1
|
||||
# via pyannote-pipeline
|
||||
packaging==23.2
|
||||
# via
|
||||
# -r requirements.in
|
||||
# black
|
||||
# huggingface-hub
|
||||
# librosa
|
||||
# lightning
|
||||
# lightning-utilities
|
||||
# marshmallow
|
||||
# matplotlib
|
||||
# mido
|
||||
# onnxruntime
|
||||
# optuna
|
||||
# pooch
|
||||
# pytest
|
||||
# pytorch-lightning
|
||||
# speechbrain
|
||||
# tensorboardx
|
||||
# tensorflow
|
||||
# torch-pitch-shift
|
||||
# torchmetrics
|
||||
# transformers
|
||||
pandas==2.2.2
|
||||
# via
|
||||
# pyannote-database
|
||||
# pyannote-metrics
|
||||
# whisperx
|
||||
pathspec==0.12.1
|
||||
# via black
|
||||
pillow==10.4.0
|
||||
# via
|
||||
# imageio
|
||||
# matplotlib
|
||||
platformdirs==4.2.2
|
||||
# via
|
||||
# black
|
||||
# pooch
|
||||
# pylint
|
||||
pluggy==1.5.0
|
||||
# via pytest
|
||||
pooch==1.8.2
|
||||
# via librosa
|
||||
pretty-midi==0.2.10
|
||||
# via -r requirements.in
|
||||
primepy==1.3
|
||||
# via torch-pitch-shift
|
||||
protobuf==3.19.6
|
||||
# via
|
||||
# onnxruntime
|
||||
# tensorboard
|
||||
# tensorboardx
|
||||
# tensorflow
|
||||
pyannote-audio==3.3.2
|
||||
# via whisperx
|
||||
pyannote-core==5.0.0
|
||||
# via
|
||||
# pyannote-audio
|
||||
# pyannote-database
|
||||
# pyannote-metrics
|
||||
# pyannote-pipeline
|
||||
pyannote-database==5.1.0
|
||||
# via
|
||||
# pyannote-audio
|
||||
# pyannote-metrics
|
||||
# pyannote-pipeline
|
||||
pyannote-metrics==3.2.1
|
||||
# via pyannote-audio
|
||||
pyannote-pipeline==3.0.1
|
||||
# via pyannote-audio
|
||||
pyasn1==0.6.0
|
||||
# via
|
||||
# pyasn1-modules
|
||||
# rsa
|
||||
pyasn1-modules==0.4.0
|
||||
# via google-auth
|
||||
pycparser==2.22
|
||||
# via cffi
|
||||
pydub==0.25.1
|
||||
# via -r requirements.in
|
||||
pygments==2.18.0
|
||||
# via rich
|
||||
pyhyphen-reloaded==4.0.6
|
||||
# via -r requirements.in
|
||||
pylint==3.2.6
|
||||
# via -r requirements.in
|
||||
pyparsing==3.1.2
|
||||
# via matplotlib
|
||||
pyreadline3==3.4.1
|
||||
# via humanfriendly
|
||||
pytest==8.3.2
|
||||
# via -r requirements.in
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# matplotlib
|
||||
# pandas
|
||||
python-levenshtein==0.25.1
|
||||
# via -r requirements.in
|
||||
pytorch-lightning==2.3.3
|
||||
# via lightning
|
||||
pytorch-metric-learning==2.6.0
|
||||
# via pyannote-audio
|
||||
pytz==2024.1
|
||||
# via pandas
|
||||
pyyaml==6.0.1
|
||||
# via
|
||||
# ctranslate2
|
||||
# demucs
|
||||
# huggingface-hub
|
||||
# hyperpyyaml
|
||||
# lightning
|
||||
# omegaconf
|
||||
# optuna
|
||||
# pyannote-database
|
||||
# pyannote-pipeline
|
||||
# pytorch-lightning
|
||||
# transformers
|
||||
rapidfuzz==3.9.4
|
||||
# via levenshtein
|
||||
regex==2024.7.24
|
||||
# via
|
||||
# nltk
|
||||
# transformers
|
||||
requests==2.32.3
|
||||
# via
|
||||
# huggingface-hub
|
||||
# music21
|
||||
# pooch
|
||||
# pyhyphen-reloaded
|
||||
# requests-oauthlib
|
||||
# tensorboard
|
||||
# transformers
|
||||
requests-oauthlib==2.0.0
|
||||
# via google-auth-oauthlib
|
||||
resampy==0.2.2
|
||||
# via
|
||||
# crepe
|
||||
# librosa
|
||||
retrying==1.3.4
|
||||
# via dora-search
|
||||
rich==13.7.1
|
||||
# via
|
||||
# pyannote-audio
|
||||
# typer
|
||||
rsa==4.9
|
||||
# via google-auth
|
||||
ruamel-yaml==0.18.6
|
||||
# via hyperpyyaml
|
||||
ruamel-yaml-clib==0.2.8
|
||||
# via ruamel-yaml
|
||||
safetensors==0.4.3
|
||||
# via transformers
|
||||
scikit-learn==1.5.1
|
||||
# via
|
||||
# crepe
|
||||
# hmmlearn
|
||||
# librosa
|
||||
# pyannote-metrics
|
||||
# pyannote-pipeline
|
||||
# pytorch-metric-learning
|
||||
scipy==1.14.1
|
||||
# via
|
||||
# -r requirements.in
|
||||
# crepe
|
||||
# hmmlearn
|
||||
# librosa
|
||||
# pyannote-core
|
||||
# pyannote-metrics
|
||||
# resampy
|
||||
# scikit-learn
|
||||
# speechbrain
|
||||
semver==3.0.2
|
||||
# via pyannote-audio
|
||||
sentencepiece==0.2.0
|
||||
# via speechbrain
|
||||
shellingham==1.5.4
|
||||
# via typer
|
||||
six==1.16.0
|
||||
# via
|
||||
# astunparse
|
||||
# google-pasta
|
||||
# keras-preprocessing
|
||||
# pretty-midi
|
||||
# python-dateutil
|
||||
# resampy
|
||||
# retrying
|
||||
# tensorflow
|
||||
sortedcontainers==2.4.0
|
||||
# via pyannote-core
|
||||
soundfile==0.12.1
|
||||
# via
|
||||
# librosa
|
||||
# pyannote-audio
|
||||
speechbrain==1.0.0
|
||||
# via pyannote-audio
|
||||
sqlalchemy==2.0.31
|
||||
# via
|
||||
# alembic
|
||||
# optuna
|
||||
submitit==1.5.1
|
||||
# via dora-search
|
||||
sympy==1.13.1
|
||||
# via
|
||||
# onnxruntime
|
||||
# pyannote-metrics
|
||||
# torch
|
||||
tabulate==0.9.0
|
||||
# via pyannote-metrics
|
||||
tensorboard==2.10.1
|
||||
# via tensorflow
|
||||
tensorboard-data-server==0.6.1
|
||||
# via tensorboard
|
||||
tensorboard-plugin-wit==1.8.1
|
||||
# via tensorboard
|
||||
tensorboardx==2.6
|
||||
# via pyannote-audio
|
||||
tensorflow==2.10.0
|
||||
# via -r requirements.in
|
||||
tensorflow-estimator==2.10.0
|
||||
# via tensorflow
|
||||
tensorflow-io-gcs-filesystem==0.31.0
|
||||
# via tensorflow
|
||||
termcolor==2.4.0
|
||||
# via tensorflow
|
||||
threadpoolctl==3.5.0
|
||||
# via scikit-learn
|
||||
tokenizers==0.15.2
|
||||
# via
|
||||
# faster-whisper
|
||||
# transformers
|
||||
tomli==2.0.1
|
||||
# via
|
||||
# black
|
||||
# pylint
|
||||
# pytest
|
||||
tomlkit==0.13.0
|
||||
# via pylint
|
||||
torch==2.4.0
|
||||
# via
|
||||
# asteroid-filterbanks
|
||||
# demucs
|
||||
# dora-search
|
||||
# julius
|
||||
# lightning
|
||||
# openunmix
|
||||
# pyannote-audio
|
||||
# pytorch-lightning
|
||||
# pytorch-metric-learning
|
||||
# speechbrain
|
||||
# torch-audiomentations
|
||||
# torch-pitch-shift
|
||||
# torchaudio
|
||||
# torchmetrics
|
||||
# whisperx
|
||||
torch-audiomentations==0.11.1
|
||||
# via pyannote-audio
|
||||
torch-pitch-shift==1.2.4
|
||||
# via torch-audiomentations
|
||||
torchaudio==2.4.0
|
||||
# via
|
||||
# demucs
|
||||
# openunmix
|
||||
# pyannote-audio
|
||||
# speechbrain
|
||||
# torch-audiomentations
|
||||
# torch-pitch-shift
|
||||
# whisperx
|
||||
torchmetrics==1.4.0.post0
|
||||
# via
|
||||
# lightning
|
||||
# pyannote-audio
|
||||
# pytorch-lightning
|
||||
tqdm==4.67.1
|
||||
# via
|
||||
# -r requirements.in
|
||||
# demucs
|
||||
# faster-whisper
|
||||
# huggingface-hub
|
||||
# lightning
|
||||
# nltk
|
||||
# openunmix
|
||||
# optuna
|
||||
# pyannote-pipeline
|
||||
# pytorch-lightning
|
||||
# pytorch-metric-learning
|
||||
# speechbrain
|
||||
# transformers
|
||||
transformers==4.39.3
|
||||
# via whisperx
|
||||
treetable==0.2.5
|
||||
# via dora-search
|
||||
typer==0.12.3
|
||||
# via pyannote-database
|
||||
typing-extensions==4.12.2
|
||||
# via
|
||||
# alembic
|
||||
# asteroid-filterbanks
|
||||
# astroid
|
||||
# black
|
||||
# huggingface-hub
|
||||
# lightning
|
||||
# lightning-utilities
|
||||
# pyannote-core
|
||||
# pytorch-lightning
|
||||
# sqlalchemy
|
||||
# submitit
|
||||
# tensorflow
|
||||
# torch
|
||||
# typer
|
||||
# typing-inspect
|
||||
typing-inspect==0.9.0
|
||||
# via dataclasses-json
|
||||
tzdata==2024.1
|
||||
# via pandas
|
||||
unidecode==1.3.8
|
||||
# via -r requirements.in
|
||||
urllib3==2.2.2
|
||||
# via requests
|
||||
webcolors==24.6.0
|
||||
# via music21
|
||||
werkzeug==3.0.3
|
||||
# via tensorboard
|
||||
wheel==0.43.0
|
||||
# via
|
||||
# astunparse
|
||||
# pyhyphen-reloaded
|
||||
# tensorboard
|
||||
whisperx @ git+https://github.com/m-bain/whisperx.git
|
||||
# via -r requirements.in
|
||||
wrapt==1.16.0
|
||||
# via tensorflow
|
||||
yarl==1.9.4
|
||||
# via aiohttp
|
||||
yt-dlp==2024.12.23
|
||||
# via -r requirements.in
|
||||
|
||||
# The following packages are considered to be unsafe in a requirements file:
|
||||
# setuptools
|
||||
+21
-1
@@ -1,2 +1,22 @@
|
||||
#!/bin/bash
|
||||
gnome-terminal -- bash -c ".venv\Scripts\activate; cd src; exec bash"
|
||||
# Activate virtual environment and run UltraSinger
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
# Check if .venv exists
|
||||
if [ ! -d ".venv" ]; then
|
||||
echo "Error: Virtual environment not found at .venv"
|
||||
echo "Please run one of the installation scripts first:"
|
||||
echo " - install/CPU/linux_cpu.sh (for CPU)"
|
||||
echo " - install/CUDA/linux_cuda_gpu.sh (for GPU with CUDA)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Activate virtual environment
|
||||
source .venv/bin/activate
|
||||
|
||||
# Navigate to src and start UltraSinger
|
||||
cd src
|
||||
echo "Starting UltraSinger..."
|
||||
python UltraSinger.py
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
# Activate virtual environment and run UltraSinger on macOS
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
# Check if .venv exists
|
||||
if [ ! -d ".venv" ]; then
|
||||
echo "Error: Virtual environment not found at .venv"
|
||||
echo "Please run the installation script first:"
|
||||
echo " - install/CPU/macos_cpu.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Activate virtual environment
|
||||
source .venv/bin/activate
|
||||
|
||||
# Navigate to src and start UltraSinger
|
||||
cd src
|
||||
echo "Starting UltraSinger..."
|
||||
python UltraSinger.py
|
||||
+4
-1
@@ -1,2 +1,5 @@
|
||||
@echo off
|
||||
start cmd /k ".venv\Scripts\activate && cd src"
|
||||
:: Activate the virtual environment and open cmd in the src directory
|
||||
call .venv\Scripts\activate.bat
|
||||
cd src
|
||||
cmd /k
|
||||
+3
-8
@@ -11,7 +11,7 @@ from modules.Ultrastar.ultrastar_txt import FormatVersion
|
||||
@dataclass
|
||||
class Settings:
|
||||
|
||||
APP_VERSION = "0.0.13-dev6"
|
||||
APP_VERSION = "0.0.13.dev16"
|
||||
CONFIDENCE_THRESHOLD = 0.6
|
||||
CONFIDENCE_PROMPT_TIMEOUT = 4
|
||||
|
||||
@@ -26,6 +26,7 @@ class Settings:
|
||||
keep_cache = False
|
||||
interactive_mode = False
|
||||
user_ffmpeg_path = ""
|
||||
quantize_to_key = True # Quantize notes to the detected key of the song
|
||||
|
||||
# Process data Paths
|
||||
input_file_path = ""
|
||||
@@ -46,16 +47,10 @@ class Settings:
|
||||
whisper_compute_type = None # change to "int8" if low on GPU mem (may reduce accuracy)
|
||||
keep_numbers = False
|
||||
|
||||
# Pitch
|
||||
crepe_model_capacity = "full" # tiny|small|medium|large|full
|
||||
crepe_step_size = 10 # in miliseconds
|
||||
|
||||
# Device
|
||||
pytorch_device = 'cpu' # cpu|cuda
|
||||
tensorflow_device = 'cpu' # cpu|cuda
|
||||
force_cpu = False
|
||||
force_whisper_cpu = False
|
||||
force_crepe_cpu = False
|
||||
|
||||
# MuseScore
|
||||
musescore_path = None
|
||||
@@ -65,7 +60,7 @@ class Settings:
|
||||
|
||||
# UltraSinger Evaluation Configuration
|
||||
test_songs_input_folder = None
|
||||
cache_override_path = None
|
||||
cache_override_path = None #"C:\\UltraSinger\\test_output"
|
||||
skip_cache_vocal_separation = False
|
||||
skip_cache_denoise_vocal_audio = False
|
||||
skip_cache_transcription = False
|
||||
|
||||
+205
-52
@@ -5,6 +5,7 @@ import getopt
|
||||
import os
|
||||
import sys
|
||||
import Levenshtein
|
||||
import librosa
|
||||
|
||||
from packaging import version
|
||||
|
||||
@@ -16,9 +17,10 @@ from modules.Audio.vocal_chunks import (
|
||||
create_audio_chunks_from_transcribed_data,
|
||||
create_audio_chunks_from_ultrastar_data,
|
||||
)
|
||||
from modules.Audio.key_detector import detect_key_from_audio, get_allowed_notes_for_key
|
||||
from modules.Audio.silence_processing import remove_silence_from_transcription_data, mute_no_singing_parts
|
||||
from modules.Audio.separation import DemucsModel
|
||||
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_wav_to_mp3
|
||||
from modules.Audio.convert_audio import convert_audio_to_mono_wav, convert_audio_format
|
||||
from modules.Audio.youtube import (
|
||||
download_from_youtube,
|
||||
)
|
||||
@@ -41,7 +43,7 @@ from modules.Midi.midi_creator import (
|
||||
from modules.Midi.MidiSegment import MidiSegment
|
||||
from modules.Midi.note_length_calculator import get_thirtytwo_note_second, get_sixteenth_note_second
|
||||
from modules.Pitcher.pitcher import (
|
||||
get_pitch_with_crepe_file,
|
||||
get_pitch_with_file,
|
||||
)
|
||||
from modules.Pitcher.pitched_data import PitchedData
|
||||
from modules.Speech_Recognition.TranscriptionResult import TranscriptionResult
|
||||
@@ -67,7 +69,12 @@ from modules.sheet import create_sheet
|
||||
from modules.ProcessData import ProcessData, ProcessDataPaths, MediaInfo
|
||||
from modules.DeviceDetection.device_detection import check_gpu_support
|
||||
from modules.Image.image_helper import save_image
|
||||
from modules.ffmpeg_helper import is_ffmpeg_available, get_ffmpeg_and_ffprobe_paths
|
||||
from modules.ffmpeg_helper import (
|
||||
is_ffmpeg_available,
|
||||
get_ffmpeg_and_ffprobe_paths,
|
||||
is_video_file,
|
||||
separate_audio_video,
|
||||
)
|
||||
|
||||
from Settings import Settings
|
||||
|
||||
@@ -150,6 +157,8 @@ def run() -> tuple[str, Score, Score]:
|
||||
print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Vocals will not be separated')}")
|
||||
if not settings.hyphenation:
|
||||
print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Hyphenation will not be applied')}")
|
||||
if settings.quantize_to_key:
|
||||
print(f"{ULTRASINGER_HEAD} {bright_green_highlighted('Option:')} {cyan_highlighted('Notes will be quantized to the detected musical key')}")
|
||||
|
||||
process_data = InitProcessData()
|
||||
|
||||
@@ -162,13 +171,23 @@ def run() -> tuple[str, Score, Score]:
|
||||
# Create process audio
|
||||
process_data.process_data_paths.processing_audio_path = CreateProcessAudio(process_data)
|
||||
|
||||
# Get BPM from wav file
|
||||
if not settings.input_file_is_ultrastar_txt:
|
||||
process_data.media_info.bpm = get_bpm_from_file(process_data.process_data_paths.processing_audio_path)
|
||||
|
||||
# Detect key
|
||||
detected_key, detected_mode = detect_key_from_audio(process_data.process_data_paths.processing_audio_path)
|
||||
if process_data.media_info.music_key is None:
|
||||
process_data.media_info.music_key = f"{detected_key} {detected_mode}"
|
||||
|
||||
# Audio transcription
|
||||
process_data.media_info.language = settings.language
|
||||
if not settings.ignore_audio:
|
||||
TranscribeAudio(process_data)
|
||||
|
||||
# Split syllables into segments
|
||||
process_data.transcribed_data = split_syllables_into_segments(process_data.transcribed_data,
|
||||
if not settings.ignore_audio:
|
||||
process_data.transcribed_data = split_syllables_into_segments(process_data.transcribed_data,
|
||||
process_data.media_info.bpm)
|
||||
|
||||
# Create audio chunks
|
||||
@@ -178,17 +197,25 @@ def run() -> tuple[str, Score, Score]:
|
||||
# Pitch audio
|
||||
process_data.pitched_data = pitch_audio(process_data.process_data_paths)
|
||||
|
||||
# Allowed keys for quantization
|
||||
allowed_notes_for_key = None
|
||||
if settings.quantize_to_key and not settings.ignore_audio:
|
||||
allowed_notes_for_key = get_allowed_notes_for_key(detected_key, detected_mode)
|
||||
|
||||
# Create Midi_Segments
|
||||
if not settings.ignore_audio:
|
||||
process_data.midi_segments = create_midi_segments_from_transcribed_data(process_data.transcribed_data,
|
||||
process_data.pitched_data)
|
||||
process_data.midi_segments = create_midi_segments_from_transcribed_data(
|
||||
process_data.transcribed_data,
|
||||
process_data.pitched_data,
|
||||
allowed_notes_for_key
|
||||
)
|
||||
else:
|
||||
process_data.midi_segments = create_repitched_midi_segments_from_ultrastar_txt(process_data.pitched_data,
|
||||
process_data.parsed_file)
|
||||
|
||||
# Merge syllable segments
|
||||
#tuple[list[MidiSegment], list[TranscribedData]]:
|
||||
process_data.midi_segments, process_data.transcribed_data = merge_syllable_segments(process_data.midi_segments,
|
||||
if not settings.ignore_audio:
|
||||
process_data.midi_segments, process_data.transcribed_data = merge_syllable_segments(process_data.midi_segments,
|
||||
process_data.transcribed_data,
|
||||
process_data.media_info.bpm)
|
||||
|
||||
@@ -221,14 +248,21 @@ def run() -> tuple[str, Score, Score]:
|
||||
def split_syllables_into_segments(
|
||||
transcribed_data: list[TranscribedData],
|
||||
real_bpm: float) -> list[TranscribedData]:
|
||||
"""Split every syllable into sub-segments"""
|
||||
"""Split every syllable into sub-segments
|
||||
|
||||
This splits long syllables (including hyphenated ones) into smaller segments
|
||||
to allow detect for pitch changes within a syllable (e.g., singing a scale).
|
||||
"""
|
||||
syllable_segment_size = get_sixteenth_note_second(real_bpm)
|
||||
thirtytwo_note = get_thirtytwo_note_second(real_bpm)
|
||||
|
||||
segment_size_decimal_points = len(str(syllable_segment_size).split(".")[1])
|
||||
new_data = []
|
||||
|
||||
for i, data in enumerate(transcribed_data):
|
||||
duration = data.end - data.start
|
||||
|
||||
# If duration is less than or equal to a 16th note, don't split
|
||||
if duration <= syllable_segment_size:
|
||||
new_data.append(data)
|
||||
continue
|
||||
@@ -259,7 +293,9 @@ def split_syllables_into_segments(
|
||||
segment.is_word_end = False
|
||||
new_data.append(segment)
|
||||
|
||||
if partial_segment >= 0.01:
|
||||
# Only add a partial_segment if it's at least as long as a 32nd note
|
||||
# Otherwise add it to the last note
|
||||
if partial_segment >= thirtytwo_note:
|
||||
first_segment.is_hyphen = True
|
||||
segment = TranscribedData()
|
||||
segment.word = "~"
|
||||
@@ -270,6 +306,9 @@ def split_syllables_into_segments(
|
||||
segment.is_hyphen = True
|
||||
segment.is_word_end = False
|
||||
new_data.append(segment)
|
||||
elif full_segments >= 1 or len(new_data) > 0:
|
||||
# Add remaining time to the last note
|
||||
new_data[-1].end += partial_segment
|
||||
|
||||
if has_space:
|
||||
new_data[-1].word += " "
|
||||
@@ -280,10 +319,17 @@ def split_syllables_into_segments(
|
||||
def merge_syllable_segments(midi_segments: list[MidiSegment],
|
||||
transcribed_data: list[TranscribedData],
|
||||
real_bpm: float) -> tuple[list[MidiSegment], list[TranscribedData]]:
|
||||
"""Merge sub-segments of a syllable where the pitch is the same"""
|
||||
"""Merge sub-segments of a syllable where the pitch is the same
|
||||
|
||||
This function handles three cases:
|
||||
1. Merge consecutive ~ segments with the SAME pitch (same note held)
|
||||
2. Detect and merge SLIDES: short ~ segments with ±1-2 semitone jumps between syllables
|
||||
3. Merge ANY consecutive segments (including regular syllables) with the SAME pitch into one word
|
||||
"""
|
||||
|
||||
thirtytwo_note = get_thirtytwo_note_second(real_bpm)
|
||||
sixteenth_note = get_sixteenth_note_second(real_bpm)
|
||||
# Slides are typically very short (1-2 16th notes)
|
||||
max_slide_duration = sixteenth_note * 2
|
||||
|
||||
new_data = []
|
||||
new_midi_notes = []
|
||||
@@ -291,26 +337,104 @@ def merge_syllable_segments(midi_segments: list[MidiSegment],
|
||||
previous_data = None
|
||||
|
||||
for i, data in enumerate(transcribed_data):
|
||||
is_note_short = data.end - data.start < thirtytwo_note
|
||||
is_same_note = midi_segments[i].note == midi_segments[i - 1].note
|
||||
# Check if previous element exists
|
||||
is_same_note = i > 0 and midi_segments[i].note == midi_segments[i - 1].note
|
||||
has_breath_pause = False
|
||||
|
||||
if previous_data is not None:
|
||||
has_breath_pause = data.start - previous_data.end > sixteenth_note
|
||||
has_breath_pause = (data.start - previous_data.end) > sixteenth_note
|
||||
|
||||
if (str(data.word).startswith("~")
|
||||
and previous_data is not None
|
||||
and (is_note_short or is_same_note)
|
||||
and not has_breath_pause):
|
||||
# Check if current word is a ~ segment (continuation marker)
|
||||
current_word_stripped = str(data.word).strip()
|
||||
is_tilde_segment = current_word_stripped == "~"
|
||||
|
||||
# Slide detection: Detect short ~ segments with small pitch jumps
|
||||
is_potential_slide = False
|
||||
if i > 0 and is_tilde_segment:
|
||||
duration = data.end - data.start
|
||||
|
||||
# Calculate pitch jump in semitones
|
||||
try:
|
||||
prev_midi = librosa.note_to_midi(midi_segments[i - 1].note)
|
||||
curr_midi = librosa.note_to_midi(midi_segments[i].note)
|
||||
semitone_diff = abs(curr_midi - prev_midi)
|
||||
|
||||
# Slide: Short duration AND small pitch jump (1-2 semitones)
|
||||
is_potential_slide = (duration <= max_slide_duration and
|
||||
semitone_diff <= 2 and
|
||||
semitone_diff > 0)
|
||||
except:
|
||||
# Ignore slide detection on error
|
||||
pass
|
||||
|
||||
# Check if current segment should be merged with previous due to same pitch
|
||||
should_merge_same_pitch = False
|
||||
if (i > 0 and
|
||||
previous_data is not None and
|
||||
not is_tilde_segment and # Not a ~ segment
|
||||
is_same_note and
|
||||
not has_breath_pause and
|
||||
not previous_data.is_word_end): # Don't merge across word boundaries
|
||||
should_merge_same_pitch = True
|
||||
|
||||
should_merge_tilde = (is_tilde_segment and
|
||||
previous_data is not None and
|
||||
(is_same_note or is_potential_slide) and
|
||||
not has_breath_pause)
|
||||
|
||||
if should_merge_tilde:
|
||||
new_data[-1].end = data.end
|
||||
new_midi_notes[-1].end = data.end
|
||||
|
||||
# For slides: Keep the original note (not the transition note)
|
||||
if is_potential_slide and not is_same_note:
|
||||
new_midi_notes[-1].note = midi_segments[i - 1].note
|
||||
|
||||
# Take over space and word_end flag from current segment
|
||||
# "~ " means end of word - add space to previous segment
|
||||
if str(data.word).endswith(" "):
|
||||
new_data[-1].word += " "
|
||||
new_midi_notes[-1].word += " "
|
||||
if not new_data[-1].word.endswith(" "):
|
||||
new_data[-1].word += " "
|
||||
if not new_midi_notes[-1].word.endswith(" "):
|
||||
new_midi_notes[-1].word += " "
|
||||
new_data[-1].is_word_end = True
|
||||
|
||||
elif should_merge_same_pitch:
|
||||
# Merge regular syllable with previous syllable (same pitch)
|
||||
new_data[-1].end = data.end
|
||||
new_midi_notes[-1].end = data.end
|
||||
|
||||
# Check if current word has space at end before stripping
|
||||
has_space = str(data.word).endswith(" ")
|
||||
|
||||
# Concatenate the words (remove trailing space from previous, add current word)
|
||||
prev_word = new_data[-1].word.rstrip()
|
||||
curr_word = data.word.rstrip()
|
||||
|
||||
# Remove ~ from BOTH previous and current word if present
|
||||
if prev_word == "~":
|
||||
prev_word = ""
|
||||
if curr_word.startswith("~"):
|
||||
curr_word = curr_word[1:]
|
||||
|
||||
# Concatenate
|
||||
new_data[-1].word = prev_word + curr_word
|
||||
new_midi_notes[-1].word = prev_word + curr_word
|
||||
|
||||
# Preserve space at end if current segment had it
|
||||
if has_space:
|
||||
new_data[-1].word += " "
|
||||
new_midi_notes[-1].word += " "
|
||||
|
||||
if data.is_word_end:
|
||||
new_data[-1].is_word_end = True
|
||||
new_data[-1].is_hyphen = False
|
||||
else:
|
||||
# Keep hyphen status if either segment was hyphenated
|
||||
new_data[-1].is_hyphen = new_data[-1].is_hyphen or data.is_hyphen
|
||||
|
||||
else:
|
||||
# Add as new segment
|
||||
new_data.append(data)
|
||||
new_midi_notes.append(midi_segments[i])
|
||||
|
||||
@@ -340,10 +464,12 @@ def InitProcessData():
|
||||
settings.output_folder_path,
|
||||
audio_file_path,
|
||||
ultrastar_class,
|
||||
audio_extension,
|
||||
) = parse_ultrastar_txt(settings.input_file_path, settings.output_folder_path)
|
||||
process_data = from_ultrastar_txt(ultrastar_class)
|
||||
process_data.basename = basename
|
||||
process_data.process_data_paths.audio_output_file_path = audio_file_path
|
||||
process_data.media_info.audio_extension = audio_extension
|
||||
# todo: ignore transcribe
|
||||
settings.ignore_audio = True
|
||||
|
||||
@@ -358,7 +484,7 @@ def InitProcessData():
|
||||
process_data.media_info
|
||||
) = download_from_youtube(settings.input_file_path, settings.output_folder_path, settings.cookiefile)
|
||||
else:
|
||||
# Audio File
|
||||
# Audio/Video File
|
||||
print(f"{ULTRASINGER_HEAD} {gold_highlighted('Full Automatic Mode')}")
|
||||
process_data = ProcessData()
|
||||
(
|
||||
@@ -366,7 +492,7 @@ def InitProcessData():
|
||||
settings.output_folder_path,
|
||||
process_data.process_data_paths.audio_output_file_path,
|
||||
process_data.media_info,
|
||||
) = infos_from_audio_input_file()
|
||||
) = infos_from_audio_video_input_file()
|
||||
return process_data
|
||||
|
||||
|
||||
@@ -397,15 +523,15 @@ def CreateUltraStarTxt(process_data: ProcessData):
|
||||
# Move instrumental and vocals
|
||||
if settings.create_karaoke and version.parse(settings.format_version.value) < version.parse(
|
||||
FormatVersion.V1_1_0.value):
|
||||
karaoke_output_path = os.path.join(settings.output_folder_path, process_data.basename + " [Karaoke].mp3")
|
||||
convert_wav_to_mp3(process_data.process_data_paths.instrumental_audio_file_path, karaoke_output_path)
|
||||
karaoke_output_path = os.path.join(settings.output_folder_path, process_data.basename + " [Karaoke]." + process_data.media_info.audio_extension)
|
||||
convert_audio_format(process_data.process_data_paths.instrumental_audio_file_path, karaoke_output_path)
|
||||
|
||||
if version.parse(settings.format_version.value) >= version.parse(FormatVersion.V1_1_0.value):
|
||||
instrumental_output_path = os.path.join(settings.output_folder_path,
|
||||
process_data.basename + " [Instrumental].mp3")
|
||||
convert_wav_to_mp3(process_data.process_data_paths.instrumental_audio_file_path, instrumental_output_path)
|
||||
vocals_output_path = os.path.join(settings.output_folder_path, process_data.basename + " [Vocals].mp3")
|
||||
convert_wav_to_mp3(process_data.process_data_paths.vocals_audio_file_path, vocals_output_path)
|
||||
process_data.basename + " [Instrumental]." + process_data.media_info.audio_extension)
|
||||
convert_audio_format(process_data.process_data_paths.instrumental_audio_file_path, instrumental_output_path)
|
||||
vocals_output_path = os.path.join(settings.output_folder_path, process_data.basename + " [Vocals]." + process_data.media_info.audio_extension)
|
||||
convert_audio_format(process_data.process_data_paths.vocals_audio_file_path, vocals_output_path)
|
||||
|
||||
# Create Ultrastar txt
|
||||
if not settings.ignore_audio:
|
||||
@@ -489,15 +615,17 @@ def transcribe_audio(cache_folder_path: str, processing_audio_path: str) -> Tran
|
||||
transcription_result = None
|
||||
whisper_align_model_string = None
|
||||
if settings.transcriber == "whisper":
|
||||
if not settings.whisper_align_model is None: whisper_align_model_string = settings.whisper_align_model.replace("/", "_")
|
||||
transcription_config = f"{settings.transcriber}_{settings.whisper_model.value}_{settings.pytorch_device}_{whisper_align_model_string}_{settings.whisper_batch_size}_{settings.whisper_compute_type}_{settings.language}"
|
||||
if not settings.whisper_align_model is None:
|
||||
whisper_align_model_string = settings.whisper_align_model.replace("/", "_")
|
||||
whisper_device = "cpu" if settings.force_whisper_cpu else settings.pytorch_device
|
||||
transcription_config = f"{settings.transcriber}_{settings.whisper_model.value}_{whisper_device}_{whisper_align_model_string}_{settings.whisper_batch_size}_{settings.whisper_compute_type}_{settings.language}"
|
||||
transcription_path = os.path.join(cache_folder_path, f"{transcription_config}.json")
|
||||
cached_transcription_available = check_file_exists(transcription_path)
|
||||
if settings.skip_cache_transcription or not cached_transcription_available:
|
||||
transcription_result = transcribe_with_whisper(
|
||||
processing_audio_path,
|
||||
settings.whisper_model,
|
||||
settings.pytorch_device,
|
||||
whisper_device,
|
||||
settings.whisper_align_model,
|
||||
settings.whisper_batch_size,
|
||||
settings.whisper_compute_type,
|
||||
@@ -516,8 +644,8 @@ def transcribe_audio(cache_folder_path: str, processing_audio_path: str) -> Tran
|
||||
return transcription_result
|
||||
|
||||
|
||||
def infos_from_audio_input_file() -> tuple[str, str, str, MediaInfo]:
|
||||
"""Infos from audio input file"""
|
||||
def infos_from_audio_video_input_file() -> tuple[str, str, str, MediaInfo]:
|
||||
"""Infos from audio/video input file"""
|
||||
basename = os.path.basename(settings.input_file_path)
|
||||
basename_without_ext = os.path.splitext(basename)[0]
|
||||
|
||||
@@ -529,27 +657,52 @@ def infos_from_audio_input_file() -> tuple[str, str, str, MediaInfo]:
|
||||
|
||||
song_info = search_musicbrainz(title, artist)
|
||||
basename_without_ext = f"{song_info.artist} - {song_info.title}"
|
||||
extension = os.path.splitext(basename)[1]
|
||||
basename = f"{basename_without_ext}{extension}"
|
||||
|
||||
song_folder_output_path = os.path.join(settings.output_folder_path, basename_without_ext)
|
||||
song_folder_output_path = get_unused_song_output_dir(song_folder_output_path)
|
||||
os_helper.create_folder(song_folder_output_path)
|
||||
os_helper.copy(settings.input_file_path, song_folder_output_path)
|
||||
os_helper.rename(
|
||||
os.path.join(song_folder_output_path, os.path.basename(settings.input_file_path)),
|
||||
os.path.join(song_folder_output_path, basename),
|
||||
)
|
||||
|
||||
extension = os.path.splitext(basename)[1]
|
||||
if is_video_file(settings.input_file_path):
|
||||
print(f"{ULTRASINGER_HEAD} Video file detected - separating audio and video")
|
||||
|
||||
video_with_audio_basename = f"{basename_without_ext}{extension}"
|
||||
video_with_audio_path = os.path.join(song_folder_output_path, video_with_audio_basename)
|
||||
os_helper.copy(settings.input_file_path, video_with_audio_path)
|
||||
|
||||
# Separate audio and video
|
||||
ultrastar_audio_input_path, final_video_path, audio_ext, video_ext = separate_audio_video(
|
||||
video_with_audio_path, basename_without_ext, song_folder_output_path
|
||||
)
|
||||
else:
|
||||
# Audio file
|
||||
basename_with_ext = f"{basename_without_ext}{extension}"
|
||||
audio_ext = extension.lstrip('.')
|
||||
video_ext = None
|
||||
os_helper.copy(settings.input_file_path, song_folder_output_path)
|
||||
os_helper.rename(
|
||||
os.path.join(song_folder_output_path, os.path.basename(settings.input_file_path)),
|
||||
os.path.join(song_folder_output_path, basename_with_ext),
|
||||
)
|
||||
ultrastar_audio_input_path = os.path.join(song_folder_output_path, basename_with_ext)
|
||||
|
||||
# Todo: Read ID3 tags
|
||||
if song_info.cover_image_data is not None:
|
||||
save_image(song_info.cover_image_data, basename_without_ext, song_folder_output_path)
|
||||
ultrastar_audio_input_path = os.path.join(song_folder_output_path, basename)
|
||||
real_bpm = get_bpm_from_file(settings.input_file_path)
|
||||
|
||||
return (
|
||||
basename_without_ext,
|
||||
song_folder_output_path,
|
||||
ultrastar_audio_input_path,
|
||||
MediaInfo(artist=song_info.artist, title=song_info.title, year=song_info.year, genre=song_info.genres, bpm=real_bpm, cover_url=song_info.cover_url),
|
||||
MediaInfo(
|
||||
artist=song_info.artist,
|
||||
title=song_info.title,
|
||||
year=song_info.year,
|
||||
genre=song_info.genres,
|
||||
cover_url=song_info.cover_url,
|
||||
audio_extension=audio_ext,
|
||||
video_extension=video_ext
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -557,16 +710,13 @@ def pitch_audio(
|
||||
process_data_paths: ProcessDataPaths) -> PitchedData:
|
||||
"""Pitch audio"""
|
||||
|
||||
pitching_config = f"crepe_{settings.ignore_audio}_{settings.crepe_model_capacity}_{settings.crepe_step_size}_{settings.tensorflow_device}"
|
||||
pitching_config = f"swiftf0_{settings.ignore_audio}"
|
||||
pitched_data_path = os.path.join(process_data_paths.cache_folder_path, f"{pitching_config}.json")
|
||||
cache_available = check_file_exists(pitched_data_path)
|
||||
|
||||
if settings.skip_cache_transcription or not cache_available:
|
||||
pitched_data = get_pitch_with_crepe_file(
|
||||
process_data_paths.processing_audio_path,
|
||||
settings.crepe_model_capacity,
|
||||
settings.crepe_step_size,
|
||||
settings.tensorflow_device,
|
||||
if settings.skip_cache_pitch_detection or not cache_available:
|
||||
pitched_data = get_pitch_with_file(
|
||||
process_data_paths.processing_audio_path
|
||||
)
|
||||
|
||||
pitched_data_json = pitched_data.to_json()
|
||||
@@ -594,7 +744,7 @@ def main(argv: list[str]) -> None:
|
||||
|
||||
def check_requirements() -> None:
|
||||
if not settings.force_cpu:
|
||||
settings.tensorflow_device, settings.pytorch_device = check_gpu_support()
|
||||
settings.pytorch_device = check_gpu_support()
|
||||
print(f"{ULTRASINGER_HEAD} ----------------------")
|
||||
|
||||
if not is_ffmpeg_available(settings.user_ffmpeg_path):
|
||||
@@ -701,6 +851,8 @@ def init_settings(argv: list[str]) -> Settings:
|
||||
settings.cookiefile = arg
|
||||
elif opt in ("--interactive"):
|
||||
settings.interactive_mode = True
|
||||
elif opt in ("--quantize_to_key"):
|
||||
settings.quantize_to_key = arg
|
||||
elif opt in ("--ffmpeg"):
|
||||
settings.user_ffmpeg_path = arg
|
||||
if settings.output_folder_path == "":
|
||||
@@ -741,6 +893,7 @@ def arg_options():
|
||||
"keep_cache",
|
||||
"musescore_path=",
|
||||
"keep_numbers",
|
||||
"quantize_to_key",
|
||||
"interactive",
|
||||
"cookiefile=",
|
||||
"ffmpeg="
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import librosa
|
||||
import soundfile as sf
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted
|
||||
|
||||
@@ -6,7 +7,7 @@ from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted
|
||||
def get_bpm_from_data(data, sampling_rate):
|
||||
"""Get real bpm from audio data"""
|
||||
onset_env = librosa.onset.onset_strength(y=data, sr=sampling_rate)
|
||||
wav_tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sampling_rate)
|
||||
wav_tempo = librosa.feature.tempo(onset_envelope=onset_env, sr=sampling_rate)
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} BPM is {blue_highlighted(str(round(wav_tempo[0], 2)))}")
|
||||
return wav_tempo[0]
|
||||
@@ -14,5 +15,11 @@ def get_bpm_from_data(data, sampling_rate):
|
||||
|
||||
def get_bpm_from_file(wav_file: str) -> float:
|
||||
"""Get real bpm from audio file"""
|
||||
data, sampling_rate = librosa.load(wav_file, sr=None)
|
||||
data, sampling_rate = sf.read(wav_file, dtype='float32')
|
||||
# Transpose if stereo to match librosa's expected format
|
||||
if len(data.shape) > 1:
|
||||
data = data.T
|
||||
# Convert to mono if stereo
|
||||
if data.ndim > 1:
|
||||
data = librosa.to_mono(data)
|
||||
return get_bpm_from_data(data, sampling_rate)
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Convert audio to other formats"""
|
||||
|
||||
from pydub import AudioSegment
|
||||
import subprocess
|
||||
import os
|
||||
import librosa
|
||||
import soundfile as sf
|
||||
|
||||
@@ -14,9 +15,27 @@ def convert_audio_to_mono_wav(input_file_path: str, output_file_path: str) -> No
|
||||
sf.write(output_file_path, y, sr)
|
||||
|
||||
|
||||
def convert_wav_to_mp3(input_file_path: str, output_file_path: str) -> None:
|
||||
"""Convert wav to mp3"""
|
||||
print(f"{ULTRASINGER_HEAD} Converting wav to mp3. -> {output_file_path}")
|
||||
def convert_audio_format(input_file_path: str, output_file_path: str) -> None:
|
||||
"""Convert audio to the format specified by the output file extension using ffmpeg"""
|
||||
output_format = os.path.splitext(output_file_path)[1].lstrip('.')
|
||||
|
||||
sound = AudioSegment.from_wav(input_file_path)
|
||||
sound.export(output_file_path, format="mp3")
|
||||
print(f"{ULTRASINGER_HEAD} Converting audio to {output_format}. -> {output_file_path}")
|
||||
# todo: makes it sense to reencode here? Its only used for Instrumental and Vocal
|
||||
# Use ffmpeg for audio conversion
|
||||
# -i: input file
|
||||
# -y: overwrite output file without asking
|
||||
# -loglevel error: only show errors
|
||||
# -q:a 0: best quality for VBR formats (mp3, ogg)
|
||||
# -codec:a copy would be fastest but only works if formats match
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-i", input_file_path,
|
||||
"-y",
|
||||
"-loglevel", "error",
|
||||
"-q:a", "0",
|
||||
output_file_path
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"FFmpeg audio conversion failed: {result.stderr}")
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Key detection and pitch quantization to musical scale"""
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
|
||||
from modules.console_colors import (
|
||||
ULTRASINGER_HEAD,
|
||||
blue_highlighted)
|
||||
|
||||
# scales (in semitones relative to root note)
|
||||
MAJOR_SCALE = [0, 2, 4, 5, 7, 9, 11]
|
||||
MINOR_SCALE = [0, 2, 3, 5, 7, 8, 10]
|
||||
NOTE_NAMES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B']
|
||||
|
||||
|
||||
def detect_key_from_audio(audio_path: str) -> tuple[str, str]:
|
||||
"""
|
||||
Detect the key and mode (major/minor) of a song from audio file.
|
||||
|
||||
Args:
|
||||
audio_path: Path to audio file
|
||||
|
||||
Returns:
|
||||
Tuple of (key_note, mode) e.g., ('C', 'major') or ('A', 'minor')
|
||||
"""
|
||||
print(f"{ULTRASINGER_HEAD} Detecting musical key")
|
||||
|
||||
y, sr = librosa.load(audio_path, sr=None, duration=60.0) # Analyze first 60 seconds
|
||||
chroma = librosa.feature.chroma_cqt(y=y, sr=sr)
|
||||
chroma_avg = np.mean(chroma, axis=1)
|
||||
chroma_avg = chroma_avg / np.sum(chroma_avg)
|
||||
|
||||
major_template = np.array([1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1]) # Major scale pattern
|
||||
minor_template = np.array([1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0]) # Minor scale pattern
|
||||
major_template = major_template / np.sum(major_template)
|
||||
minor_template = minor_template / np.sum(minor_template)
|
||||
|
||||
best_correlation = -1
|
||||
best_key = None
|
||||
best_mode = None
|
||||
|
||||
# Try all 12 keys
|
||||
for shift in range(12):
|
||||
shifted_major = np.roll(major_template, shift)
|
||||
shifted_minor = np.roll(minor_template, shift)
|
||||
|
||||
major_corr = np.corrcoef(chroma_avg, shifted_major)[0, 1]
|
||||
minor_corr = np.corrcoef(chroma_avg, shifted_minor)[0, 1]
|
||||
|
||||
if major_corr > best_correlation:
|
||||
best_correlation = major_corr
|
||||
best_key = NOTE_NAMES[shift]
|
||||
best_mode = 'major'
|
||||
|
||||
if minor_corr > best_correlation:
|
||||
best_correlation = minor_corr
|
||||
best_key = NOTE_NAMES[shift]
|
||||
best_mode = 'minor'
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Detected key: {blue_highlighted(best_key)} {blue_highlighted(best_mode)}")
|
||||
return best_key, best_mode
|
||||
|
||||
|
||||
def get_allowed_notes_for_key(key_note: str, mode: str) -> set[str]:
|
||||
"""
|
||||
Get all allowed note names for a given key across all octaves.
|
||||
|
||||
Args:
|
||||
key_note: Root note (e.g., 'C', 'D#', 'F')
|
||||
mode: 'major' or 'minor'
|
||||
|
||||
Returns:
|
||||
Set of allowed note names (e.g., {'C', 'D', 'E', ...})
|
||||
"""
|
||||
root_idx = NOTE_NAMES.index(key_note)
|
||||
scale = MAJOR_SCALE if mode == 'major' else MINOR_SCALE
|
||||
|
||||
allowed_notes = set()
|
||||
for interval in scale:
|
||||
note_idx = (root_idx + interval) % 12
|
||||
allowed_notes.add(NOTE_NAMES[note_idx])
|
||||
|
||||
return allowed_notes
|
||||
|
||||
|
||||
def quantize_note_to_key(note: str, allowed_notes: set[str]) -> str:
|
||||
"""
|
||||
Quantize a note to the nearest allowed note in the key.
|
||||
|
||||
Args:
|
||||
note: MIDI note name (e.g., 'C4', 'D#5')
|
||||
allowed_notes: Set of allowed note names without octave
|
||||
|
||||
Returns:
|
||||
Quantized note name
|
||||
"""
|
||||
# Parse note and octave
|
||||
if len(note) == 0:
|
||||
return note
|
||||
|
||||
# Extract note name and octave
|
||||
if len(note) >= 2 and note[-2] == '#':
|
||||
note_name = note[:-1]
|
||||
octave = note[-1]
|
||||
else:
|
||||
note_name = note[:-1] if note[-1].isdigit() else note
|
||||
octave = note[-1] if note[-1].isdigit() else ''
|
||||
|
||||
# If already in key, return as is
|
||||
if note_name in allowed_notes:
|
||||
return note
|
||||
|
||||
# Find nearest allowed note
|
||||
try:
|
||||
note_midi = librosa.note_to_midi(note)
|
||||
except:
|
||||
return note
|
||||
|
||||
# Try all allowed notes in nearby octaves
|
||||
min_distance = float('inf')
|
||||
best_note = note
|
||||
|
||||
for allowed_note_name in allowed_notes:
|
||||
# Try same octave and adjacent octaves
|
||||
for oct_offset in [-1, 0, 1]:
|
||||
if not octave:
|
||||
test_octave = 4 + oct_offset
|
||||
else:
|
||||
test_octave = int(octave) + oct_offset
|
||||
|
||||
if test_octave < 0 or test_octave > 8:
|
||||
continue
|
||||
|
||||
test_note = f"{allowed_note_name}{test_octave}"
|
||||
|
||||
try:
|
||||
test_midi = librosa.note_to_midi(test_note)
|
||||
distance = abs(test_midi - note_midi)
|
||||
|
||||
if distance < min_distance:
|
||||
min_distance = distance
|
||||
best_note = test_note
|
||||
except:
|
||||
continue
|
||||
|
||||
#print(f"{ULTRASINGER_HEAD} Moved {blue_highlighted(note)} to {blue_highlighted(best_note)}")
|
||||
return best_note
|
||||
@@ -10,6 +10,7 @@ from modules.Audio.bpm import get_bpm_from_file
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
from modules.Image.image_helper import save_image
|
||||
from modules.musicbrainz_client import search_musicbrainz
|
||||
from modules.ffmpeg_helper import separate_audio_video
|
||||
|
||||
|
||||
def get_youtube_title(url: str, cookiefile: str = None) -> tuple[str, str]:
|
||||
@@ -30,20 +31,18 @@ def get_youtube_title(url: str, cookiefile: str = None) -> tuple[str, str]:
|
||||
return result["channel"].strip(), result["title"].strip()
|
||||
|
||||
|
||||
def __download_youtube_audio(url: str, clear_filename: str, output_path: str, cookiefile: str = None):
|
||||
"""Download audio from YouTube"""
|
||||
def __download_youtube_video_with_audio(url: str, clear_filename: str, output_path: str, cookiefile: str = None) -> str:
|
||||
"""Download video with audio from YouTube and return the file extension"""
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Downloading Audio")
|
||||
print(f"{ULTRASINGER_HEAD} Downloading Video with Audio")
|
||||
ydl_opts = {
|
||||
"format": "bestaudio/best",
|
||||
"outtmpl": output_path + "/" + clear_filename,
|
||||
"postprocessors": [
|
||||
{"key": "FFmpegExtractAudio", "preferredcodec": "mp3"}
|
||||
],
|
||||
"format": "bestvideo[ext=mp4]+bestaudio/best",
|
||||
"outtmpl": output_path + "/" + clear_filename + ".%(ext)s",
|
||||
"merge_output_format": "mp4",
|
||||
"cookiefile": cookiefile,
|
||||
}
|
||||
|
||||
__start_download(ydl_opts, url)
|
||||
return "mp4"
|
||||
|
||||
|
||||
def __download_youtube_thumbnail(url: str, clear_filename: str, output_path: str, cookiefile: str = None) -> str:
|
||||
@@ -75,16 +74,6 @@ def download_and_convert_thumbnail(ydl_opts, url: str, clear_filename: str, outp
|
||||
return ""
|
||||
|
||||
|
||||
def __download_youtube_video(url: str, clear_filename: str, output_path: str, cookiefile: str = None) -> None:
|
||||
"""Download video from YouTube"""
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Downloading Video")
|
||||
ydl_opts = {
|
||||
"format": "bestvideo[ext=mp4]/mp4",
|
||||
"outtmpl": output_path + "/" + clear_filename + ".mp4",
|
||||
"cookiefile": cookiefile,
|
||||
}
|
||||
__start_download(ydl_opts, url)
|
||||
|
||||
|
||||
def __start_download(ydl_opts, url: str) -> None:
|
||||
@@ -104,26 +93,41 @@ def download_from_youtube(input_url: str, output_folder_path: str, cookiefile: s
|
||||
song_info = search_musicbrainz(title, artist)
|
||||
|
||||
basename_without_ext = sanitize_filename(f"{song_info.artist} - {song_info.title}")
|
||||
basename = basename_without_ext + ".mp3"
|
||||
song_output = os.path.join(output_folder_path, basename_without_ext)
|
||||
song_output = get_unused_song_output_dir(song_output)
|
||||
os_helper.create_folder(song_output)
|
||||
__download_youtube_audio(input_url, basename_without_ext, song_output, cookiefile)
|
||||
__download_youtube_video(input_url, basename_without_ext, song_output, cookiefile)
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Downloading from YouTube")
|
||||
video_ext = __download_youtube_video_with_audio(
|
||||
input_url, basename_without_ext, song_output, cookiefile
|
||||
)
|
||||
video_with_audio_path = os.path.join(song_output, f"{basename_without_ext}.{video_ext}")
|
||||
|
||||
# Separate audio and video
|
||||
audio_file_path, final_video_path, audio_ext, video_ext = separate_audio_video(
|
||||
video_with_audio_path, basename_without_ext, song_output
|
||||
)
|
||||
|
||||
if song_info.cover_url is not None and song_info.cover_image_data is not None:
|
||||
cover_url = song_info.cover_url
|
||||
save_image(song_info.cover_image_data, basename_without_ext, song_output)
|
||||
else:
|
||||
cover_url = __download_youtube_thumbnail(
|
||||
input_url, basename_without_ext, song_output
|
||||
)
|
||||
audio_file_path = os.path.join(song_output, basename)
|
||||
real_bpm = get_bpm_from_file(audio_file_path)
|
||||
input_url, basename_without_ext, song_output, cookiefile
|
||||
)
|
||||
|
||||
return (
|
||||
basename_without_ext,
|
||||
song_output,
|
||||
audio_file_path,
|
||||
MediaInfo(artist=song_info.artist, title=song_info.title, year=song_info.year, genre=song_info.genres, bpm=real_bpm,
|
||||
cover_url=cover_url, video_url=input_url),
|
||||
MediaInfo(
|
||||
artist=song_info.artist,
|
||||
title=song_info.title,
|
||||
year=song_info.year,
|
||||
genre=song_info.genres,
|
||||
cover_url=cover_url,
|
||||
video_url=input_url,
|
||||
audio_extension=audio_ext,
|
||||
video_extension=video_ext
|
||||
),
|
||||
)
|
||||
@@ -1,38 +1,19 @@
|
||||
"""Device detection module."""
|
||||
|
||||
import torch
|
||||
import os
|
||||
import tensorflow as tf
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD, red_highlighted, blue_highlighted
|
||||
|
||||
tensorflow_gpu_supported = False
|
||||
pytorch_gpu_supported = False
|
||||
|
||||
def check_gpu_support() -> tuple[bool, bool]:
|
||||
"""Check worker device (e.g cuda or cpu) supported by tensorflow and pytorch"""
|
||||
def check_gpu_support() -> str:
|
||||
"""Check worker device (e.g cuda or cpu) supported by pytorch"""
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Checking GPU support.")
|
||||
|
||||
pytorch_gpu_supported = __check_pytorch_support()
|
||||
tensorflow_gpu_supported = __check_tensorflow_support()
|
||||
|
||||
return 'cuda' if tensorflow_gpu_supported else 'cpu', 'cuda' if pytorch_gpu_supported else 'cpu'
|
||||
|
||||
|
||||
def __check_tensorflow_support():
|
||||
tensorflow_gpu_supported = False
|
||||
gpus = tf.config.list_physical_devices('GPU')
|
||||
if gpus:
|
||||
tensorflow_gpu_supported = True
|
||||
print(f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - using {red_highlighted('cuda')} gpu.")
|
||||
else:
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - there are no {red_highlighted('cuda')} devices available -> Using {red_highlighted('cpu')}.")
|
||||
if os.name == 'nt':
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} {blue_highlighted('tensorflow')} - versions above 2.10 dropped GPU support for Windows, refer to the readme for possible solutions.")
|
||||
return tensorflow_gpu_supported
|
||||
return 'cuda' if pytorch_gpu_supported else 'cpu'
|
||||
|
||||
|
||||
def __check_pytorch_support():
|
||||
|
||||
@@ -17,7 +17,7 @@ from modules.console_colors import (
|
||||
from modules.Ultrastar.ultrastar_txt import UltrastarTxtValue
|
||||
from modules.Pitcher.pitched_data import PitchedData
|
||||
from modules.Pitcher.pitched_data_helper import get_frequencies_with_high_confidence
|
||||
|
||||
from modules.Audio.key_detector import quantize_note_to_key
|
||||
|
||||
def create_midi_instrument(midi_segments: list[MidiSegment]) -> object:
|
||||
"""Converts an Ultrastar data to a midi instrument"""
|
||||
@@ -83,9 +83,20 @@ def find_nearest_index(array: list[float], value: float) -> int:
|
||||
return idx
|
||||
|
||||
|
||||
def create_midi_notes_from_pitched_data(start_times: list[float], end_times: list[float], words: list[str], pitched_data: PitchedData) -> list[
|
||||
MidiSegment]:
|
||||
"""Create midi notes from pitched data"""
|
||||
def create_midi_notes_from_pitched_data(start_times: list[float], end_times: list[float], words: list[str],
|
||||
pitched_data: PitchedData, allowed_notes: set[str] = None) -> list[MidiSegment]:
|
||||
"""Create midi notes from pitched data
|
||||
|
||||
Args:
|
||||
start_times: List of start times
|
||||
end_times: List of end times
|
||||
words: List of words/syllables
|
||||
pitched_data: Pitched data containing frequencies and confidence
|
||||
allowed_notes: Optional set of allowed note names for key quantization
|
||||
|
||||
Returns:
|
||||
List of MidiSegments
|
||||
"""
|
||||
print(f"{ULTRASINGER_HEAD} Creating midi_segments")
|
||||
|
||||
midi_segments = []
|
||||
@@ -94,7 +105,7 @@ def create_midi_notes_from_pitched_data(start_times: list[float], end_times: lis
|
||||
end_time = end_times[index]
|
||||
word = str(words[index])
|
||||
|
||||
midi_segment = create_midi_note_from_pitched_data(start_time, end_time, pitched_data, word)
|
||||
midi_segment = create_midi_note_from_pitched_data(start_time, end_time, pitched_data, word, allowed_notes)
|
||||
midi_segments.append(midi_segment)
|
||||
|
||||
# todo: Progress?
|
||||
@@ -102,8 +113,20 @@ def create_midi_notes_from_pitched_data(start_times: list[float], end_times: lis
|
||||
return midi_segments
|
||||
|
||||
|
||||
def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitched_data: PitchedData, word: str) -> MidiSegment:
|
||||
"""Create midi note from pitched data"""
|
||||
def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitched_data: PitchedData, word: str,
|
||||
allowed_notes: set[str] = None) -> MidiSegment:
|
||||
"""Create midi note from pitched data
|
||||
|
||||
Args:
|
||||
start_time: Start time of the note
|
||||
end_time: End time of the note
|
||||
pitched_data: Pitched data containing frequencies and confidence
|
||||
word: The word/syllable for this note
|
||||
allowed_notes: Optional set of allowed note names (without octave) for key quantization
|
||||
|
||||
Returns:
|
||||
One MidiSegment
|
||||
"""
|
||||
|
||||
start = find_nearest_index(pitched_data.times, start_time)
|
||||
end = find_nearest_index(pitched_data.times, end_time)
|
||||
@@ -121,10 +144,24 @@ def create_midi_note_from_pitched_data(start_time: float, end_time: float, pitch
|
||||
|
||||
note = most_frequent(notes)[0][0]
|
||||
|
||||
if allowed_notes is not None:
|
||||
note = quantize_note_to_key(note, allowed_notes)
|
||||
|
||||
return MidiSegment(note, start_time, end_time, word)
|
||||
|
||||
|
||||
def create_midi_segments_from_transcribed_data(transcribed_data: list[TranscribedData], pitched_data: PitchedData) -> list[MidiSegment]:
|
||||
def create_midi_segments_from_transcribed_data(transcribed_data: list[TranscribedData], pitched_data: PitchedData,
|
||||
allowed_notes: set[str] = None) -> list[MidiSegment]:
|
||||
"""Create MIDI segments from transcribed data
|
||||
|
||||
Args:
|
||||
transcribed_data: List of transcribed data segments
|
||||
pitched_data: Pitched data containing frequencies and confidence
|
||||
allowed_notes: Optional set of allowed note names for key quantization
|
||||
|
||||
Returns:
|
||||
List of MidiSegments
|
||||
"""
|
||||
start_times = []
|
||||
end_times = []
|
||||
words = []
|
||||
@@ -135,7 +172,7 @@ def create_midi_segments_from_transcribed_data(transcribed_data: list[Transcribe
|
||||
end_times.append(midi_segment.end)
|
||||
words.append(midi_segment.word)
|
||||
midi_segments = create_midi_notes_from_pitched_data(start_times, end_times, words,
|
||||
pitched_data)
|
||||
pitched_data, allowed_notes)
|
||||
return midi_segments
|
||||
|
||||
|
||||
|
||||
@@ -1,41 +1,74 @@
|
||||
"""Pitcher module"""
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
import crepe
|
||||
from scipy.io import wavfile
|
||||
from swift_f0 import SwiftF0
|
||||
|
||||
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
|
||||
from modules.Midi.midi_creator import convert_frequencies_to_notes, most_frequent
|
||||
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted
|
||||
from modules.Pitcher.pitched_data import PitchedData
|
||||
from modules.Pitcher.pitched_data_helper import get_frequencies_with_high_confidence
|
||||
|
||||
_swift_f0_detector = None
|
||||
|
||||
def _get_detector():
|
||||
"""Lazy initialize SwiftF0 detector"""
|
||||
global _swift_f0_detector
|
||||
if _swift_f0_detector is None:
|
||||
# Initialize for general music/speech (wide frequency range) fmin=46.875, fmax=2093.75
|
||||
# For speech only: fmin=65, fmax=400
|
||||
_swift_f0_detector = SwiftF0(fmin=46.875, fmax=2093.75, confidence_threshold=0.9)
|
||||
return _swift_f0_detector
|
||||
|
||||
|
||||
def get_pitch_with_crepe_file(
|
||||
filename: str, model_capacity: str, step_size: int = 10, device: str = "cpu"
|
||||
def get_pitch_with_file(
|
||||
filename: str
|
||||
) -> PitchedData:
|
||||
"""Pitch with crepe"""
|
||||
"""Pitch detection using SwiftF0"""
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('crepe')} and model {blue_highlighted(model_capacity)} and {red_highlighted(device)} as worker"
|
||||
f"{ULTRASINGER_HEAD} Pitching with {blue_highlighted('SwiftF0')}"
|
||||
)
|
||||
sample_rate, audio = wavfile.read(filename)
|
||||
|
||||
return get_pitch_with_crepe(audio, sample_rate, model_capacity, step_size)
|
||||
# Convert stereo to mono if needed
|
||||
if len(audio.shape) > 1:
|
||||
audio = np.mean(audio, axis=1)
|
||||
|
||||
# Normalize audio to float32 based on dtype
|
||||
if audio.dtype == np.uint8:
|
||||
# uint8: range [0, 255] -> subtract 128 and divide by 128
|
||||
audio = (audio.astype(np.float32) - 128.0) / 128.0
|
||||
elif audio.dtype in [np.int16, np.int32, np.int64]:
|
||||
# Signed integers: use iinfo to get max value and normalize
|
||||
dtype_info = np.iinfo(audio.dtype)
|
||||
max_val = max(abs(dtype_info.min), abs(dtype_info.max))
|
||||
audio = audio.astype(np.float32) / float(max_val)
|
||||
elif audio.dtype == np.float64:
|
||||
# float64: cast to float32
|
||||
audio = audio.astype(np.float32)
|
||||
elif audio.dtype != np.float32:
|
||||
# Fallback for other types: assume int16 range
|
||||
audio = audio.astype(np.float32) / 32768.0
|
||||
|
||||
return get_pitch_with_swift_f0(audio, sample_rate)
|
||||
|
||||
|
||||
def get_pitch_with_crepe(
|
||||
audio, sample_rate: int, model_capacity: str, step_size: int = 10
|
||||
def get_pitch_with_swift_f0(
|
||||
audio: np.ndarray, sample_rate: int
|
||||
) -> PitchedData:
|
||||
"""Pitch with crepe"""
|
||||
"""Pitch detection using SwiftF0
|
||||
|
||||
# Info: The model is trained on 16 kHz audio, so if the input audio has a different sample rate, it will be first resampled to 16 kHz using resampy inside crepe.
|
||||
SwiftF0 processes audio at 16kHz with 256-sample hop size internally.
|
||||
Returns frames at approximately 62.5 ms intervals.
|
||||
"""
|
||||
detector = _get_detector()
|
||||
|
||||
times, frequencies, confidence, activation = crepe.predict(
|
||||
audio, sample_rate, model_capacity, step_size=step_size, viterbi=True
|
||||
)
|
||||
# Detect pitch
|
||||
result = detector.detect_from_array(audio, sample_rate)
|
||||
|
||||
# convert to native float for serialization
|
||||
confidence = [float(x) for x in confidence]
|
||||
# Convert to PitchedData format
|
||||
times = [float(t) for t in result.timestamps]
|
||||
frequencies = [float(f) for f in result.pitch_hz]
|
||||
confidence = [float(c) for c in result.confidence]
|
||||
|
||||
return PitchedData(times, frequencies, confidence)
|
||||
|
||||
@@ -53,39 +86,6 @@ def get_pitched_data_with_high_confidence(
|
||||
|
||||
return new_pitched_data
|
||||
|
||||
# Todo: Unused
|
||||
def pitch_each_chunk_with_crepe(directory: str,
|
||||
crepe_model_capacity: str,
|
||||
crepe_step_size: int,
|
||||
tensorflow_device: str) -> list[str]:
|
||||
"""Pitch each chunk with crepe and return midi notes"""
|
||||
print(f"{ULTRASINGER_HEAD} Pitching each chunk with {blue_highlighted('crepe')}")
|
||||
|
||||
midi_notes = []
|
||||
for filename in sorted(
|
||||
[f for f in os.listdir(directory) if f.endswith(".wav")],
|
||||
key=lambda x: int(x.split("_")[1]),
|
||||
):
|
||||
filepath = os.path.join(directory, filename)
|
||||
# todo: stepsize = duration? then when shorter than "it" it should take the duration. Otherwise there a more notes
|
||||
pitched_data = get_pitch_with_crepe_file(
|
||||
filepath,
|
||||
crepe_model_capacity,
|
||||
crepe_step_size,
|
||||
tensorflow_device,
|
||||
)
|
||||
conf_f = get_frequencies_with_high_confidence(
|
||||
pitched_data.frequencies, pitched_data.confidence
|
||||
)
|
||||
|
||||
notes = convert_frequencies_to_notes(conf_f)
|
||||
note = most_frequent(notes)[0][0]
|
||||
|
||||
midi_notes.append(note)
|
||||
# todo: Progress?
|
||||
# print(filename + " f: " + str(mean))
|
||||
|
||||
return midi_notes
|
||||
|
||||
class Pitcher:
|
||||
"""Docstring"""
|
||||
@@ -19,17 +19,20 @@ class MediaInfo:
|
||||
"""Media Info"""
|
||||
title: str
|
||||
artist: str
|
||||
bpm: float
|
||||
bpm: Optional[float] = None
|
||||
year: Optional[str] = None
|
||||
genre: Optional[str] = None
|
||||
language: Optional[str] = None
|
||||
cover_url: Optional[str] = None
|
||||
video_url: Optional[str] = None
|
||||
music_key: Optional[str] = None # (e.g., "C major", "A minor")
|
||||
video_extension: Optional[str] = None # (e.g., "mp4", "mkv")
|
||||
audio_extension: Optional[str] = None # (e.g., "mp3", "m4a")
|
||||
|
||||
@dataclass
|
||||
class ProcessData:
|
||||
"""Data for processing"""
|
||||
process_data_paths: ProcessDataPaths = ProcessDataPaths()
|
||||
process_data_paths: ProcessDataPaths = field(default_factory=ProcessDataPaths)
|
||||
basename: Optional[str] = None
|
||||
media_info: Optional[MediaInfo] = None
|
||||
transcribed_data: Optional[List[TranscribedData]] = field(default_factory=list)
|
||||
|
||||
@@ -1,7 +1,26 @@
|
||||
"""Whisper Speech Recognition Module"""
|
||||
import os
|
||||
import inspect
|
||||
import textwrap
|
||||
|
||||
# Set environment variable to handle cuDNN loading issues
|
||||
# CUDA_MODULE_LOADING=LAZY allows PyTorch to continue even if some cuDNN modules are not found
|
||||
#os.environ['CUDA_MODULE_LOADING'] = 'LAZY'
|
||||
|
||||
import torch
|
||||
|
||||
# Fix for PyTorch 2.6+ compatibility with omegaconf and old models
|
||||
# MUST be done before importing whisperx or any other modules that use torch.load
|
||||
# Monkey-patch torch.load to use weights_only=False for compatibility with older models
|
||||
# This is necessary because PyTorch 2.6+ changed the default to weights_only=True
|
||||
# but WhisperX/Pyannote models were saved with pickle and require weights_only=False
|
||||
_original_torch_load = torch.load
|
||||
def _patched_torch_load(*args, **kwargs):
|
||||
# Force weights_only=False regardless of what was passed
|
||||
kwargs['weights_only'] = False
|
||||
return _original_torch_load(*args, **kwargs)
|
||||
torch.load = _patched_torch_load
|
||||
|
||||
import whisperx
|
||||
from enum import Enum
|
||||
from torch.cuda import OutOfMemoryError
|
||||
@@ -80,68 +99,15 @@ def transcribe_with_whisper(
|
||||
keep_numbers: bool = False,
|
||||
) -> TranscriptionResult:
|
||||
"""Transcribe with whisper"""
|
||||
# Info: Monkey Patch FasterWhisperPipeline.detect_language to include error handling for low confidence
|
||||
src = textwrap.dedent(inspect.getsource(whisperx.asr.FasterWhisperPipeline.detect_language))
|
||||
# Replace the relevant part of the method
|
||||
start_token = "if audio.shape[0] < N_SAMPLES:"
|
||||
end_token = "return language"
|
||||
replacement = """\
|
||||
#Added imports
|
||||
from modules.console_colors import ULTRASINGER_HEAD, blue_highlighted, red_highlighted
|
||||
from Settings import Settings
|
||||
from inputimeout import inputimeout, TimeoutOccurred
|
||||
#End Import addition
|
||||
if audio.shape[0] < N_SAMPLES:
|
||||
print("Warning: audio is shorter than 30s, language detection may be inaccurate.")
|
||||
model_n_mels = self.model.feat_kwargs.get("feature_size")
|
||||
segment = log_mel_spectrogram(audio[: N_SAMPLES],
|
||||
n_mels=model_n_mels if model_n_mels is not None else 80,
|
||||
padding=0 if audio.shape[0] >= N_SAMPLES else N_SAMPLES - audio.shape[0])
|
||||
encoder_output = self.model.encode(segment)
|
||||
results = self.model.model.detect_language(encoder_output)
|
||||
language_token, language_probability = results[0][0]
|
||||
language = language_token[2:-2]
|
||||
print(f"Detected language: {language} ({language_probability:.2f}) in first 30s of audio...")
|
||||
#Added handling for low detection probability
|
||||
if language_probability < Settings.CONFIDENCE_THRESHOLD:
|
||||
print(f"{ULTRASINGER_HEAD} {red_highlighted('Warning:')} Language detection probability for detected language {language} is below {Settings.CONFIDENCE_THRESHOLD}, results may be inaccurate.")
|
||||
print(f"{ULTRASINGER_HEAD} Override the language below or re-run with parameter {blue_highlighted('--language xx')} to specify the song language...")
|
||||
try:
|
||||
response = inputimeout(
|
||||
prompt=f"{ULTRASINGER_HEAD} Do you want to continue with {language} (default) or override with another language (y)? (y/n): ",
|
||||
timeout=Settings.CONFIDENCE_PROMPT_TIMEOUT
|
||||
).strip().lower()
|
||||
except TimeoutOccurred:
|
||||
import locale
|
||||
print(f"{ULTRASINGER_HEAD} No user input received in {Settings.CONFIDENCE_PROMPT_TIMEOUT} seconds. Attempting automatic override with system locale.")
|
||||
print(f"{ULTRASINGER_HEAD} Trying to get language from default locale...")
|
||||
current_locale = locale.getlocale()
|
||||
if current_locale[0]:
|
||||
language_code = current_locale[0][:2].strip().lower()
|
||||
print(f"{ULTRASINGER_HEAD} Found language code: {language_code} in locale. Setting language to {blue_highlighted(language_code)}...")
|
||||
language = language_code
|
||||
else:
|
||||
print(f"{ULTRASINGER_HEAD} No locale is set.")
|
||||
response = 'n'
|
||||
language_response = response == 'y'
|
||||
if language_response:
|
||||
language = input(f"{ULTRASINGER_HEAD} Please enter the language code for the language you want to use (e.g. 'en', 'de', 'es', etc.): ").strip().lower()
|
||||
#End addition
|
||||
"""
|
||||
new_src = replace_code_lines(src, start_token, end_token, replacement)
|
||||
# Compile it and execute it in the target module's namespace
|
||||
exec(compile(new_src, "<string>", "exec"), whisperx.asr.__dict__)
|
||||
whisperx.asr.FasterWhisperPipeline.detect_language = whisperx.asr.detect_language
|
||||
#End Monkey Patch
|
||||
|
||||
# Info: Regardless of the audio sampling rate used in the original audio file, whisper resample the audio signal to 16kHz (via ffmpeg). So the standard input from (44.1 or 48 kHz) should work.
|
||||
|
||||
print(
|
||||
f"{ULTRASINGER_HEAD} Loading {blue_highlighted('whisper')} with model {blue_highlighted(model.value)} and {red_highlighted(device)} as worker"
|
||||
)
|
||||
if alignment_model is not None:
|
||||
print(f"{ULTRASINGER_HEAD} using alignment model {blue_highlighted(alignment_model)}")
|
||||
|
||||
# Fixme: Why it is not none for cpu?
|
||||
#compute_type = "int8"
|
||||
if compute_type is None:
|
||||
compute_type = "float16" if device == "cuda" else "int8"
|
||||
|
||||
@@ -196,6 +162,9 @@ def transcribe_with_whisper(
|
||||
|
||||
return TranscriptionResult(transcribed_data, detected_language)
|
||||
except ValueError as value_error:
|
||||
# Restore original torch.load in case of error
|
||||
torch.load = _original_torch_load
|
||||
|
||||
if (
|
||||
"Requested float16 compute type, but the target device or backend do not support efficient float16 computation."
|
||||
in str(value_error.args[0])
|
||||
@@ -207,14 +176,24 @@ def transcribe_with_whisper(
|
||||
|
||||
raise value_error
|
||||
except OutOfMemoryError as oom_exception:
|
||||
# Restore original torch.load in case of error
|
||||
torch.load = _original_torch_load
|
||||
|
||||
print(oom_exception)
|
||||
print(MEMORY_ERROR_MESSAGE)
|
||||
raise oom_exception
|
||||
except Exception as exception:
|
||||
# Restore original torch.load in case of error
|
||||
torch.load = _original_torch_load
|
||||
|
||||
if "CUDA failed with error out of memory" in str(exception.args[0]):
|
||||
print(exception)
|
||||
print(MEMORY_ERROR_MESSAGE)
|
||||
raise exception
|
||||
finally:
|
||||
# Restore original torch.load after models are loaded
|
||||
# This ensures other modules (like pitch detection) are not affected by the monkey-patch
|
||||
torch.load = _original_torch_load
|
||||
|
||||
|
||||
def convert_to_transcribed_data(result_aligned):
|
||||
|
||||
@@ -66,11 +66,14 @@ def create_ultrastar_txt_from_automation(
|
||||
|
||||
ultrastar_txt = UltrastarTxtValue()
|
||||
ultrastar_txt.version = format_version.value
|
||||
ultrastar_txt.mp3 = basename + ".mp3"
|
||||
ultrastar_txt.audio = basename + ".mp3"
|
||||
ultrastar_txt.vocals = basename + " [Vocals].mp3"
|
||||
ultrastar_txt.instrumental = basename + " [Instrumental].mp3"
|
||||
ultrastar_txt.video = basename + ".mp4"
|
||||
if media_info.audio_extension is None:
|
||||
raise Exception("Missing Audio extension. It is required to create Ultrastar txt")
|
||||
ultrastar_txt.mp3 = basename + "." + media_info.audio_extension
|
||||
ultrastar_txt.audio = basename + "." + media_info.audio_extension
|
||||
ultrastar_txt.vocals = basename + " [Vocals]." + media_info.audio_extension
|
||||
ultrastar_txt.instrumental = basename + " [Instrumental]." + media_info.audio_extension
|
||||
if media_info.video_extension is not None:
|
||||
ultrastar_txt.video = basename + "." + media_info.video_extension
|
||||
ultrastar_txt.language = media_info.language
|
||||
cover = basename + " [CO].jpg"
|
||||
ultrastar_txt.cover = (
|
||||
@@ -90,6 +93,9 @@ def create_ultrastar_txt_from_automation(
|
||||
ultrastar_txt.videoUrl = media_info.video_url
|
||||
if media_info.cover_url is not None:
|
||||
ultrastar_txt.coverUrl = media_info.cover_url
|
||||
if media_info.music_key is not None:
|
||||
# todo: as list add here?
|
||||
ultrastar_txt.tags = f"key: {media_info.music_key}"
|
||||
|
||||
ultrastar_file_output_path = os.path.join(song_folder_output_path, basename + ".txt")
|
||||
create_ultrastar_txt(
|
||||
@@ -101,7 +107,7 @@ def create_ultrastar_txt_from_automation(
|
||||
if create_karaoke and version.parse(format_version.value) < version.parse(FormatVersion.V1_1_0.value):
|
||||
title = basename + " [Karaoke]"
|
||||
ultrastar_txt.title = title
|
||||
ultrastar_txt.mp3 = title + ".mp3"
|
||||
ultrastar_txt.mp3 = title + "." + media_info.audio_extension
|
||||
karaoke_output_path = os.path.join(song_folder_output_path, title)
|
||||
karaoke_txt_output_path = karaoke_output_path + ".txt"
|
||||
create_ultrastar_txt(
|
||||
|
||||
@@ -33,35 +33,33 @@ def parse(input_file: str) -> UltrastarTxtValue:
|
||||
for line in txt:
|
||||
count += 1
|
||||
if line.startswith("#"):
|
||||
if line.startswith(f"#{UltrastarTxtTag.ARTIST}"):
|
||||
if line.startswith(f"#{UltrastarTxtTag.ARTIST.value}"):
|
||||
ultrastar_class.artist = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.TITLE}"):
|
||||
elif line.startswith(f"#{UltrastarTxtTag.TITLE.value}"):
|
||||
ultrastar_class.title = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.MP3}"):
|
||||
elif line.startswith(f"#{UltrastarTxtTag.MP3.value}"):
|
||||
ultrastar_class.mp3 = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.AUDIO}"):
|
||||
elif line.startswith(f"#{UltrastarTxtTag.AUDIO.value}"):
|
||||
ultrastar_class.audio = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.VIDEO}"):
|
||||
elif line.startswith(f"#{UltrastarTxtTag.VIDEO.value}"):
|
||||
ultrastar_class.video = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.GAP}"):
|
||||
elif line.startswith(f"#{UltrastarTxtTag.GAP.value}"):
|
||||
ultrastar_class.gap = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.BPM}"):
|
||||
elif line.startswith(f"#{UltrastarTxtTag.BPM.value}"):
|
||||
ultrastar_class.bpm = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.VIDEO}"):
|
||||
ultrastar_class.video = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.VIDEOGAP}"):
|
||||
elif line.startswith(f"#{UltrastarTxtTag.VIDEOGAP.value}"):
|
||||
ultrastar_class.videoGap = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.COVER}"):
|
||||
elif line.startswith(f"#{UltrastarTxtTag.COVER.value}"):
|
||||
ultrastar_class.cover = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(f"#{UltrastarTxtTag.BACKGROUND}"):
|
||||
elif line.startswith(f"#{UltrastarTxtTag.BACKGROUND.value}"):
|
||||
ultrastar_class.background = line.split(":")[1].replace("\n", "")
|
||||
elif line.startswith(
|
||||
(
|
||||
f"{UltrastarTxtNoteTypeTag.FREESTYLE} ",
|
||||
f"{UltrastarTxtNoteTypeTag.NORMAL} ",
|
||||
f"{UltrastarTxtNoteTypeTag.GOLDEN} ",
|
||||
f"{UltrastarTxtNoteTypeTag.RAP} ",
|
||||
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN} ",
|
||||
f"{UltrastarTxtNoteTypeTag.FREESTYLE.value} ",
|
||||
f"{UltrastarTxtNoteTypeTag.NORMAL.value} ",
|
||||
f"{UltrastarTxtNoteTypeTag.GOLDEN.value} ",
|
||||
f"{UltrastarTxtNoteTypeTag.RAP.value} ",
|
||||
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN.value} ",
|
||||
)
|
||||
):
|
||||
parts = line.split()
|
||||
@@ -86,7 +84,7 @@ def parse(input_file: str) -> UltrastarTxtValue:
|
||||
return ultrastar_class
|
||||
|
||||
|
||||
def parse_ultrastar_txt(input_file_path: str, output_folder_path: str) -> tuple[str, str, str, UltrastarTxtValue]:
|
||||
def parse_ultrastar_txt(input_file_path: str, output_folder_path: str) -> tuple[str, str, str, UltrastarTxtValue, str]:
|
||||
"""Parse Ultrastar txt"""
|
||||
ultrastar_class = parse(input_file_path)
|
||||
|
||||
@@ -100,6 +98,8 @@ def parse_ultrastar_txt(input_file_path: str, output_folder_path: str) -> tuple[
|
||||
f"an audio file."
|
||||
)
|
||||
exit(1)
|
||||
_, audio_ext_with_dot = os.path.splitext(ultrastar_mp3_name)
|
||||
audio_ext = audio_ext_with_dot.lstrip('.')
|
||||
|
||||
song_output = os.path.join(
|
||||
output_folder_path,
|
||||
@@ -118,4 +118,5 @@ def parse_ultrastar_txt(input_file_path: str, output_folder_path: str) -> tuple[
|
||||
song_output,
|
||||
str(audio_file_path),
|
||||
ultrastar_class,
|
||||
audio_ext
|
||||
)
|
||||
@@ -43,13 +43,13 @@ class Points:
|
||||
def add_point(note_type: str, points: Points) -> Points:
|
||||
"""Add calculated points to the points object."""
|
||||
|
||||
if note_type == UltrastarTxtNoteTypeTag.NORMAL:
|
||||
if note_type == UltrastarTxtNoteTypeTag.NORMAL.value:
|
||||
points.notes += 1
|
||||
elif note_type == UltrastarTxtNoteTypeTag.GOLDEN:
|
||||
elif note_type == UltrastarTxtNoteTypeTag.GOLDEN.value:
|
||||
points.golden_notes += 2
|
||||
elif note_type == UltrastarTxtNoteTypeTag.RAP:
|
||||
elif note_type == UltrastarTxtNoteTypeTag.RAP.value:
|
||||
points.rap += 1
|
||||
elif note_type == UltrastarTxtNoteTypeTag.RAP_GOLDEN:
|
||||
elif note_type == UltrastarTxtNoteTypeTag.RAP_GOLDEN.value:
|
||||
points.golden_rap += 2
|
||||
return points
|
||||
|
||||
@@ -75,6 +75,9 @@ def get_score(points: Points) -> Score:
|
||||
if points.line_bonus == 0
|
||||
else MAX_SONG_SCORE - MAX_SONG_LINE_BONUS
|
||||
)
|
||||
if(points.parts == 0):
|
||||
print(f"{ULTRASINGER_HEAD} No parts found, returning 0 score")
|
||||
return score
|
||||
score.notes = round(
|
||||
score.max_score * (points.notes + points.rap) / points.parts
|
||||
)
|
||||
@@ -100,6 +103,9 @@ def calculate_score(pitched_data: PitchedData, ultrastar_class: UltrastarTxtValu
|
||||
simple_points = Points()
|
||||
accurate_points = Points()
|
||||
|
||||
if(len(ultrastar_class.UltrastarNoteLines) == 0):
|
||||
print(f"{ULTRASINGER_HEAD} No note lines found in Ultrastar txt, returning 0 points")
|
||||
return get_score(simple_points), get_score(accurate_points)
|
||||
reachable_line_bonus_per_word = MAX_SONG_LINE_BONUS / len(ultrastar_class.UltrastarNoteLines)
|
||||
step_size = 0.09 # Todo: Whats is the step size of the game? Its not 1/bps -> one beat in seconds s = 60/bpm
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ class UltrastarTxtTag(str, Enum):
|
||||
AUDIO = 'AUDIO' # Its instead of MP3. Just renamed
|
||||
VOCALS = 'VOCALS' # Vocals only audio
|
||||
INSTRUMENTAL = 'INSTRUMENTAL' # Instrumental only audio
|
||||
TAGS = 'TAGS' # Tags for the song. Can be used for filtering
|
||||
TAGS = 'TAGS' # Tags for the song. Can be used for filtering. Separation with ","
|
||||
|
||||
# 1.2.0
|
||||
VIDEOURL = 'VIDEOURL' # URL to the video file
|
||||
|
||||
@@ -53,42 +53,43 @@ def create_ultrastar_txt(
|
||||
gap = midi_segments[0].start
|
||||
|
||||
if version.parse(ultrastar_class.version) >= version.parse("1.0.0"):
|
||||
file.write(f"#{UltrastarTxtTag.VERSION}:{ultrastar_class.version}\n"),
|
||||
file.write(f"#{UltrastarTxtTag.ARTIST}:{ultrastar_class.artist}\n")
|
||||
file.write(f"#{UltrastarTxtTag.TITLE}:{ultrastar_class.title}\n")
|
||||
file.write(f"#{UltrastarTxtTag.VERSION.value}:{ultrastar_class.version}\n")
|
||||
file.write(f"#{UltrastarTxtTag.ARTIST.value}:{ultrastar_class.artist}\n")
|
||||
file.write(f"#{UltrastarTxtTag.TITLE.value}:{ultrastar_class.title}\n")
|
||||
if ultrastar_class.year is not None:
|
||||
file.write(f"#{UltrastarTxtTag.YEAR}:{ultrastar_class.year}\n")
|
||||
file.write(f"#{UltrastarTxtTag.YEAR.value}:{ultrastar_class.year}\n")
|
||||
if ultrastar_class.language is not None:
|
||||
file.write(f"#{UltrastarTxtTag.LANGUAGE}:{get_language_name(ultrastar_class.language)}\n")
|
||||
file.write(f"#{UltrastarTxtTag.LANGUAGE.value}:{get_language_name(ultrastar_class.language)}\n")
|
||||
if ultrastar_class.genre:
|
||||
file.write(f"#{UltrastarTxtTag.GENRE}:{ultrastar_class.genre}\n")
|
||||
file.write(f"#{UltrastarTxtTag.GENRE.value}:{ultrastar_class.genre}\n")
|
||||
if ultrastar_class.cover is not None:
|
||||
file.write(f"#{UltrastarTxtTag.COVER}:{ultrastar_class.cover}\n")
|
||||
file.write(f"#{UltrastarTxtTag.COVER.value}:{ultrastar_class.cover}\n")
|
||||
if version.parse(ultrastar_class.version) >= version.parse("1.2.0"):
|
||||
if ultrastar_class.coverUrl is not None:
|
||||
file.write(f"#{UltrastarTxtTag.COVERURL}:{ultrastar_class.coverUrl}\n")
|
||||
file.write(f"#{UltrastarTxtTag.COVERURL.value}:{ultrastar_class.coverUrl}\n")
|
||||
if ultrastar_class.background is not None:
|
||||
file.write(f"#{UltrastarTxtTag.BACKGROUND}:{ultrastar_class.background}\n")
|
||||
file.write(f"#{UltrastarTxtTag.MP3}:{ultrastar_class.mp3}\n")
|
||||
file.write(f"#{UltrastarTxtTag.BACKGROUND.value}:{ultrastar_class.background}\n")
|
||||
file.write(f"#{UltrastarTxtTag.MP3.value}:{ultrastar_class.mp3}\n")
|
||||
if version.parse(ultrastar_class.version) >= version.parse("1.1.0"):
|
||||
file.write(f"#{UltrastarTxtTag.AUDIO}:{ultrastar_class.audio}\n")
|
||||
file.write(f"#{UltrastarTxtTag.AUDIO.value}:{ultrastar_class.audio}\n")
|
||||
if ultrastar_class.vocals is not None:
|
||||
file.write(f"#{UltrastarTxtTag.VOCALS}:{ultrastar_class.vocals}\n")
|
||||
file.write(f"#{UltrastarTxtTag.VOCALS.value}:{ultrastar_class.vocals}\n")
|
||||
if ultrastar_class.instrumental is not None:
|
||||
file.write(f"#{UltrastarTxtTag.INSTRUMENTAL}:{ultrastar_class.instrumental}\n")
|
||||
if ultrastar_class.tags is not None:
|
||||
file.write(f"#{UltrastarTxtTag.TAGS}:{ultrastar_class.tags}\n")
|
||||
file.write(f"#{UltrastarTxtTag.INSTRUMENTAL.value}:{ultrastar_class.instrumental}\n")
|
||||
if ultrastar_class.video is not None:
|
||||
file.write(f"#{UltrastarTxtTag.VIDEO}:{ultrastar_class.video}\n")
|
||||
file.write(f"#{UltrastarTxtTag.VIDEO.value}:{ultrastar_class.video}\n")
|
||||
if ultrastar_class.videoGap is not None:
|
||||
file.write(f"#{UltrastarTxtTag.VIDEOGAP}:{ultrastar_class.videoGap}\n")
|
||||
file.write(f"#{UltrastarTxtTag.VIDEOGAP.value}:{ultrastar_class.videoGap}\n")
|
||||
if version.parse(ultrastar_class.version) >= version.parse("1.2.0"):
|
||||
if ultrastar_class.videoUrl is not None:
|
||||
file.write(f"#{UltrastarTxtTag.VIDEOURL}:{ultrastar_class.videoUrl}\n")
|
||||
file.write(f"#{UltrastarTxtTag.BPM}:{round(ultrastar_bpm, 2)}\n") # not the real BPM!
|
||||
file.write(f"#{UltrastarTxtTag.GAP}:{int(gap * 1000)}\n")
|
||||
file.write(f"#{UltrastarTxtTag.CREATOR}:{ultrastar_class.creator}\n")
|
||||
file.write(f"#{UltrastarTxtTag.COMMENT}:{ultrastar_class.comment}\n")
|
||||
file.write(f"#{UltrastarTxtTag.VIDEOURL.value}:{ultrastar_class.videoUrl}\n")
|
||||
file.write(f"#{UltrastarTxtTag.BPM.value}:{round(ultrastar_bpm, 2)}\n") # not the real BPM!
|
||||
file.write(f"#{UltrastarTxtTag.GAP.value}:{int(gap * 1000)}\n")
|
||||
if version.parse(ultrastar_class.version) >= version.parse("1.1.0"):
|
||||
if ultrastar_class.tags is not None:
|
||||
file.write(f"#{UltrastarTxtTag.TAGS.value}:{ultrastar_class.tags}\n")
|
||||
file.write(f"#{UltrastarTxtTag.CREATOR.value}:{ultrastar_class.creator}\n")
|
||||
file.write(f"#{UltrastarTxtTag.COMMENT.value}:{ultrastar_class.comment}\n")
|
||||
|
||||
# Write the singing part
|
||||
previous_end_beat = 0
|
||||
@@ -118,7 +119,7 @@ def create_ultrastar_txt(
|
||||
# 'n2' duration at real beat
|
||||
# 'n3' pitch where 0 == C4
|
||||
# 'w' lyric
|
||||
line = f"{UltrastarTxtNoteTypeTag.NORMAL} " \
|
||||
line = f"{UltrastarTxtNoteTypeTag.NORMAL.value} " \
|
||||
f"{str(start_beat)} " \
|
||||
f"{str(duration)} " \
|
||||
f"{str(convert_midi_note_to_ultrastar_note(midi_segment))} " \
|
||||
@@ -140,11 +141,11 @@ def create_ultrastar_txt(
|
||||
second_to_beat(midi_segment.end - gap, real_bpm)
|
||||
* multiplication
|
||||
)
|
||||
linebreak = f"{UltrastarTxtTag.LINEBREAK} " \
|
||||
linebreak = f"{UltrastarTxtTag.LINEBREAK.value} " \
|
||||
f"{str(round(show_next))}\n"
|
||||
file.write(linebreak)
|
||||
separated_word_silence = []
|
||||
file.write(f"{UltrastarTxtTag.FILE_END}")
|
||||
file.write(f"{UltrastarTxtTag.FILE_END.value}")
|
||||
|
||||
|
||||
def deviation(silence_parts):
|
||||
@@ -191,7 +192,7 @@ def create_repitched_txt_from_ultrastar_data(
|
||||
# todo: just add '_repitched' to input_file
|
||||
with open(output_repitched_ultrastar, "w", encoding=FILE_ENCODING) as file:
|
||||
for line in txt:
|
||||
if line.startswith(f"{UltrastarTxtNoteTypeTag.NORMAL} "):
|
||||
if line.startswith(f"{UltrastarTxtNoteTypeTag.NORMAL.value} "):
|
||||
parts = re.findall(r"\S+|\s+", line)
|
||||
# between are whitespaces
|
||||
# [0] :
|
||||
@@ -214,21 +215,21 @@ def add_score_to_ultrastar_txt(ultrastar_file_output: str, score: Score) -> None
|
||||
text = text.split("\n")
|
||||
|
||||
for i, line in enumerate(text):
|
||||
if line.startswith(f"#{UltrastarTxtTag.COMMENT}:"):
|
||||
if line.startswith(f"#{UltrastarTxtTag.COMMENT.value}:"):
|
||||
text[
|
||||
i
|
||||
] = f"{line} | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}"
|
||||
break
|
||||
|
||||
if line.startswith((
|
||||
f"{UltrastarTxtNoteTypeTag.FREESTYLE} ",
|
||||
f"{UltrastarTxtNoteTypeTag.NORMAL} ",
|
||||
f"{UltrastarTxtNoteTypeTag.GOLDEN} ",
|
||||
f"{UltrastarTxtNoteTypeTag.RAP} ",
|
||||
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN} ")):
|
||||
f"{UltrastarTxtNoteTypeTag.FREESTYLE.value} ",
|
||||
f"{UltrastarTxtNoteTypeTag.NORMAL.value} ",
|
||||
f"{UltrastarTxtNoteTypeTag.GOLDEN.value} ",
|
||||
f"{UltrastarTxtNoteTypeTag.RAP.value} ",
|
||||
f"{UltrastarTxtNoteTypeTag.RAP_GOLDEN.value} ")):
|
||||
text.insert(
|
||||
i,
|
||||
f"#{UltrastarTxtTag.COMMENT}: UltraSinger [GitHub] | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}",
|
||||
f"#{UltrastarTxtTag.COMMENT.value}: UltraSinger [GitHub] | Score: total: {score.score}, notes: {score.notes} line: {score.line_bonus}, golden: {score.golden}",
|
||||
)
|
||||
break
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ def print_help() -> None:
|
||||
[opt]
|
||||
-h This help text.
|
||||
-i Ultrastar.txt
|
||||
audio like .mp3, .wav, youtube link
|
||||
audio/video like .mp3, .mp4, .wav, youtube link
|
||||
-o Output folder
|
||||
|
||||
[mode]
|
||||
@@ -54,6 +54,7 @@ def print_help() -> None:
|
||||
--create_audio_chunks Enable creation of audio chunks. Audio chunks are disabled by default.
|
||||
--keep_cache Keep cache folder after creation. Cache folder is removed by default.
|
||||
--plot Enable creation of plots. Plots are disabled by default.
|
||||
--quantize_to_key Quantize notes to the detected musical key. This removes slides and out-of-key notes.
|
||||
--format_version 0.3.0|1.0.0|1.1.0|1.2.0 >> ((default) is 1.2.0)
|
||||
--musescore_path path to MuseScore executable
|
||||
--keep_numbers Transcribe numbers as digits and not words
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
|
||||
def __add_ffmpeg_to_path(ffmpeg_path) -> None:
|
||||
@@ -29,3 +30,139 @@ def is_ffmpeg_available(user_path: str = "") -> bool:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def extract_audio(input_video_path: str, output_audio_path: str) -> None:
|
||||
"""Extract audio from video file in original format without re-encoding"""
|
||||
ffmpeg_path, _ = get_ffmpeg_and_ffprobe_paths()
|
||||
|
||||
cmd = [
|
||||
ffmpeg_path,
|
||||
"-i",
|
||||
input_video_path,
|
||||
"-vn", # No video
|
||||
"-acodec",
|
||||
"copy", # Copy audio codec without re-encoding
|
||||
"-y", # Overwrite output file if exists
|
||||
output_audio_path,
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise Exception(f"FFmpeg audio extraction failed: {result.stderr}")
|
||||
|
||||
|
||||
def remove_audio_from_video(input_video_path: str, output_video_path: str) -> None:
|
||||
"""Remove audio from video file without re-encoding video"""
|
||||
ffmpeg_path, _ = get_ffmpeg_and_ffprobe_paths()
|
||||
|
||||
cmd = [
|
||||
ffmpeg_path,
|
||||
"-i",
|
||||
input_video_path,
|
||||
"-an", # No audio
|
||||
"-vcodec",
|
||||
"copy", # Copy video codec without re-encoding
|
||||
"-y", # Overwrite output file if exists
|
||||
output_video_path,
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise Exception(f"FFmpeg audio removal failed: {result.stderr}")
|
||||
|
||||
|
||||
def is_video_file(file_path: str) -> bool:
|
||||
"""Check if file contains video streams using ffprobe"""
|
||||
try:
|
||||
_, ffprobe_path = get_ffmpeg_and_ffprobe_paths()
|
||||
|
||||
cmd = [
|
||||
ffprobe_path,
|
||||
"-v", "error",
|
||||
"-select_streams", "v:0",
|
||||
"-show_entries", "stream=codec_type",
|
||||
"-of", "default=noprint_wrappers=1:nokey=1",
|
||||
file_path
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
return result.returncode == 0 and result.stdout.strip() == "video"
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def get_audio_codec_and_extension(video_file_path: str) -> str:
|
||||
"""
|
||||
Detect audio codec from video file and return codec name and appropriate file extension.
|
||||
"""
|
||||
try:
|
||||
_, ffprobe_path = get_ffmpeg_and_ffprobe_paths()
|
||||
|
||||
cmd = [
|
||||
ffprobe_path,
|
||||
"-v", "error",
|
||||
"-select_streams", "a:0",
|
||||
"-show_entries", "stream=codec_name",
|
||||
"-of", "default=noprint_wrappers=1:nokey=1",
|
||||
video_file_path
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0 or not result.stdout.strip():
|
||||
# Default to wav if detection fails
|
||||
return "wav"
|
||||
|
||||
codec_name = result.stdout.strip()
|
||||
|
||||
codec_to_extension = {
|
||||
"aac": "aac",
|
||||
"mp3": "mp3",
|
||||
"opus": "opus",
|
||||
"vorbis": "ogg",
|
||||
"flac": "flac",
|
||||
"pcm_s16le": "wav",
|
||||
"pcm_s24le": "wav",
|
||||
"pcm_s32le": "wav",
|
||||
"ac3": "ac3",
|
||||
"eac3": "eac3",
|
||||
}
|
||||
|
||||
extension = codec_to_extension.get(codec_name, "wav")
|
||||
return extension
|
||||
|
||||
except Exception:
|
||||
# Default to wav if detection fails
|
||||
return "wav"
|
||||
|
||||
|
||||
def separate_audio_video(video_with_audio_path: str, basename_without_ext: str, output_folder: str) -> tuple[str, str, str, str]:
|
||||
"""
|
||||
Separate audio and video from a video file.
|
||||
Automatically detects the audio codec and uses the appropriate file extension.
|
||||
"""
|
||||
from modules.console_colors import ULTRASINGER_HEAD
|
||||
|
||||
# Get original video file extension without the dot
|
||||
_, video_ext_with_dot = os.path.splitext(video_with_audio_path)
|
||||
video_ext = video_ext_with_dot.lstrip('.')
|
||||
|
||||
# Detect audio codec and get appropriate extension
|
||||
audio_ext = get_audio_codec_and_extension(video_with_audio_path)
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Extracting audio from video")
|
||||
audio_file_path = os.path.join(output_folder, f"{basename_without_ext}.{audio_ext}")
|
||||
extract_audio(video_with_audio_path, audio_file_path)
|
||||
|
||||
print(f"{ULTRASINGER_HEAD} Creating video without audio")
|
||||
video_only_path = os.path.join(output_folder, f"{basename_without_ext}_video.{video_ext}")
|
||||
remove_audio_from_video(video_with_audio_path, video_only_path)
|
||||
|
||||
# Remove original video with audio
|
||||
os.remove(video_with_audio_path)
|
||||
|
||||
# Rename video without audio to final name
|
||||
final_video_path = os.path.join(output_folder, f"{basename_without_ext}.{video_ext}")
|
||||
os.rename(video_only_path, final_video_path)
|
||||
|
||||
return audio_file_path, final_video_path, audio_ext, video_ext
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import musicbrainzngs
|
||||
import string
|
||||
import time
|
||||
from urllib.error import URLError
|
||||
from Levenshtein import ratio
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
@@ -25,10 +27,22 @@ title_filter = [
|
||||
]
|
||||
|
||||
|
||||
MAX_RETRIES = 3
|
||||
|
||||
|
||||
def __clean_string(s: str) -> str:
|
||||
return s.translate(str.maketrans('', '', string.punctuation)).lower().strip()
|
||||
|
||||
|
||||
def __musicbrainz_request(func):
|
||||
for i in range(MAX_RETRIES):
|
||||
try:
|
||||
return func()
|
||||
except musicbrainzngs.musicbrainz.NetworkError:
|
||||
time.sleep(1)
|
||||
return None
|
||||
|
||||
|
||||
def search_musicbrainz(title: str, artist) -> SongInfo:
|
||||
# Musicbrainz API documentation
|
||||
# https://python-musicbrainzngs.readthedocs.io/en/latest/api/
|
||||
@@ -68,8 +82,12 @@ def __single_line_search(search_string):
|
||||
search_string = __clean_string(search_string)
|
||||
search_string = __filter_words(search_string)
|
||||
|
||||
artists = musicbrainzngs.search_artists(search_string, limit=10, artist=search_string)
|
||||
recordings = musicbrainzngs.search_recordings(search_string, limit=100, artistname=search_string)
|
||||
artists = __musicbrainz_request(lambda: musicbrainzngs.search_artists(search_string, limit=10, artist=search_string))
|
||||
recordings = __musicbrainz_request(lambda: musicbrainzngs.search_recordings(search_string, limit=100, artistname=search_string))
|
||||
|
||||
if artists is None or recordings is None:
|
||||
return None
|
||||
|
||||
found_artist = None
|
||||
|
||||
for record in recordings['recording-list']:
|
||||
@@ -114,8 +132,13 @@ def __multi_line_search(artist: str, title: str):
|
||||
artist1, title1 = artist, title
|
||||
artist2, title2 = title, artist
|
||||
|
||||
result1 = musicbrainzngs.search_recordings(recording=title1, limit=10, artist=artist1, artistname=artist1)
|
||||
result2 = musicbrainzngs.search_recordings(recording=title2, limit=10, artist=artist2, artistname=artist2)
|
||||
result1 = __musicbrainz_request(lambda: musicbrainzngs.search_recordings(recording=title1, limit=10, artist=artist1, artistname=artist1))
|
||||
result2 = __musicbrainz_request(lambda: musicbrainzngs.search_recordings(recording=title2, limit=10, artist=artist2, artistname=artist2))
|
||||
|
||||
if result1 is None:
|
||||
result1 = {'recording-count': 0, 'recording-list': []}
|
||||
if result2 is None:
|
||||
result2 = {'recording-count': 0, 'recording-list': []}
|
||||
|
||||
# Filter result to ['artist-credit-phrase'] == artist
|
||||
record1 = [x for x in result1['recording-list'] if
|
||||
@@ -157,8 +180,14 @@ def __get_image(recording) -> (bytes, str):
|
||||
if 'release-list' in recording:
|
||||
for release in recording['release-list']:
|
||||
try:
|
||||
image_data = musicbrainzngs.get_image_front(release['id'])
|
||||
image_list = musicbrainzngs.get_image_list(release['id'])
|
||||
image_data = __musicbrainz_request(lambda: musicbrainzngs.get_image_front(release['id']))
|
||||
if image_data is None:
|
||||
continue
|
||||
|
||||
image_list = __musicbrainz_request(lambda: musicbrainzngs.get_image_list(release['id']))
|
||||
if image_list is None:
|
||||
continue
|
||||
|
||||
for image in image_list['images']:
|
||||
if image['front']:
|
||||
image_url = image['image']
|
||||
@@ -179,7 +208,11 @@ def __get_year(recording):
|
||||
return year
|
||||
|
||||
release_group_id = recording['release-list'][0]['release-group']['id']
|
||||
release_group = musicbrainzngs.get_release_group_by_id(release_group_id)
|
||||
release_group = __musicbrainz_request(lambda: musicbrainzngs.get_release_group_by_id(release_group_id))
|
||||
|
||||
if release_group is None:
|
||||
return year
|
||||
|
||||
if 'first-release-date' not in release_group['release-group']:
|
||||
return year
|
||||
|
||||
|
||||
Reference in new issue
Block a user