VoiceCraftFinetune.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyMME1/E11fkmRRob9c+yTqX",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Sewlell/VoiceCraft/blob/master/VoiceCraftFinetune.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5w3Pfk-EnmBr"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/jasonppy/VoiceCraft"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q condacolab\n",
        "import condacolab\n",
        "condacolab.install()\n",
        "\n",
        "# ---------------------- Currently this Colab doesn't have saving model feature yet. Only remove the below # if you wanted to add yourself.\n",
        "\n",
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "jV3Sm6xXxiOZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Rerun this cell twice. And let it restart."
      ],
      "metadata": {
        "id": "_leCoLsf534C"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -e git+https://github.com/facebookresearch/audiocraft.git@c5157b5bf14bf83449c17ea1eeb66c19fb4bc7f0#egg=audiocraft\n",
        "!pip install xformers==0.0.22\n",
        "!pip install torchaudio==2.0.2 torch==2.0.1 # this assumes your system is compatible with CUDA 11.7, otherwise checkout https://pytorch.org/get-started/previous-versions/#v201\n",
        "!apt-get install ffmpeg # if you don't already have ffmpeg installed\n",
        "!apt-get install espeak-ng # backend for the phonemizer installed below\n",
        "!pip install tensorboard==2.16.2\n",
        "!pip install phonemizer==3.2.1\n",
        "!pip install datasets==2.16.0\n",
        "!pip install torchmetrics==0.11.1\n",
        "!pip install huggingface_hub==0.22.2"
      ],
      "metadata": {
        "id": "oCTm5EHexBM5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "\n",
        "# Upload the audio file to Google Drive\n",
        "uploaded = files.upload()\n",
        "\n",
        "audio_path = next(iter(uploaded))\n",
        "\n",
        "# Print the path of the uploaded audio file\n",
        "print(f\"Uploaded audio file: {audio_path}\")\n"
      ],
      "metadata": {
        "id": "SFr-DETfbYYR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title ## Create Metadata\n",
        "\n",
        "#@markdown Size of Gigaspeech, `xl` is used for VoiceCraft training and `xs` is good for debugging\n",
        "dataset_size = \"xs\" # @param [\"xs\", \"s\", \"m\", \"l\", \"xl\"]\n",
        "#@markdown Number of samples in each mega batch for multiprocess dataloading\n",
        "mega_batch_size = 100 # @param {type:\"integer\"}\n",
        "#@markdown Number of samples in each batch for singleprocess dataloading\n",
        "number_of_worker = 4 # @param {type:\"integer\"}\n",
        "#@markdown Batch size for encodec encoding, decrease it if OOM.\n",
        "batch_size = 6 # @param {type: \"slider\", min: 1, max: 32}\n",
        "#@markdown Encodec input audio sample rate\n",
        "audio_sample_rate = \"41000\" # @param [\"16000\", \"40000\", \"41000\"] {allow-input: true}\n",
        "#@markdown [ADVANCED] Encodec downsample rate (Default: 320)\n",
        "downsample_rate = 320 # @param {type:\"integer\"}\n",
        "#@markdown [ADVANCED] Encodec model code sample rate (Default: 50)\n",
        "modelcode_sr = 50 # @param {type:\"integer\"}\n",
        "#@markdown Drop audios that are longer than this number ( in second )\n",
        "len_cap = 35.0 # @param {type:\"number\"}\n",
        "#@markdown Max length of audio in samples, if exceed, will cut a batch into half to process, decrease this number if OOM on your machine ( in second )\n",
        "max_len = 30000 # @param {type:\"integer\"}\n"
      ],
      "metadata": {
        "id": "NbenHcsyy7t3",
        "cellView": "form"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown If you use Gigaspeech dataset or other dataset that required you to sign an agreement in order to download the dataset. Please run this cell.\n",
        "\n",
        "#@markdown When it ask for your access token, give it the token which you can get it here (https://huggingface.co/settings/tokens). Also it should be read, not write.\n",
        "\n",
        "import os\n",
        "\n",
        "# Set the environment variable 'USER' to the current user's name.\n",
        "os.environ[\"USER\"] = os.getenv(\"USER\", \"user\")\n",
        "\n",
        "!huggingface-cli login"
      ],
      "metadata": {
        "cellView": "form",
        "id": "G10drdtlm-30"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import shutil\n",
        "\n",
        "!wget https://huggingface.co/pyp1/VoiceCraft/resolve/main/encodec_4cb2048_giga.th -O encodec_4cb2048_giga.th\n",
        "destination_dir = \"/content/VoiceCraft/data/\"\n",
        "os.makedirs(destination_dir, exist_ok=True)\n",
        "shutil.move(\"encodec_4cb2048_giga.th\", os.path.join(destination_dir, \"encodec_4cb2048_giga.th\"))"
      ],
      "metadata": {
        "id": "8jd_dSemfWI7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Ignore `words count mismatch` warning!\n",
        "\n",
        "### Run this if you wanted to run on Gigaspeech dataset."
      ],
      "metadata": {
        "id": "pe-Wfz7BpxYd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!python /content/VoiceCraft/data/phonemize_encodec_encode_hf.py \\\n",
        "--dataset_size {dataset_size} \\\n",
        "--download_to /content/VoiceCraft/data/scratch/pyp/datasets/gigaspeech_debug \\\n",
        "--save_dir /content/VoiceCraft/data/scratch/pyp/datasets/gigaspeech_phn_enc_manifest_debug \\\n",
        "--encodec_model_path /content/VoiceCraft/data/encodec_4cb2048_giga.th \\\n",
        "--n_workers {number_of_worker} \\\n",
        "--mega_batch_size {mega_batch_size} \\\n",
        "--batch_size {batch_size} \\\n",
        "--model_sr {audio_sample_rate} \\\n",
        "--downsample_rate {downsample_rate} \\\n",
        "--model_code_sr {modelcode_sr} \\\n",
        "--len_cap {len_cap} \\\n",
        "--max_len {max_len}\n"
      ],
      "metadata": {
        "id": "pSyajSnci7mz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import shutil\n",
        "\n",
        "# Create the destination directory if it doesn't exist\n",
        "destination_dir = \"/content/VoiceCraft/data/scratch/pyp/datasets/gigaspeech_phn_enc_manifest_debug/manifest/\"\n",
        "os.makedirs(destination_dir, exist_ok=True)\n",
        "\n",
        "# Download the file from Hugging Face\n",
        "!wget https://huggingface.co/datasets/pyp1/VoiceCraft_RealEdit/resolve/main/train.txt -O train.txt\n",
        "!wget https://huggingface.co/datasets/pyp1/VoiceCraft_RealEdit/resolve/main/validation.txt -O validation.txt\n",
        "# This as we finetune VoiceCraft pretrained\n",
        "!wget https://huggingface.co/datasets/pyp1/VoiceCraft_RealEdit/resolve/main/vocab.txt -O vocab.txt\n",
        "\n",
        "# Move the downloaded file to the destination directory\n",
        "shutil.move(\"train.txt\", os.path.join(destination_dir, \"train.txt\"))\n",
        "shutil.move(\"validation.txt\", os.path.join(destination_dir, \"validation.txt\"))\n",
        "shutil.move(\"vocab.txt\", os.path.join(destination_dir, \"vocab.txt\"))\n"
      ],
      "metadata": {
        "id": "M_jM45yROBJB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title ## Start Training\n",
        "!bash /content/VoiceCraft/z_scripts/e830M_ft.sh\n",
        "\n",
        "# ^^ Turn 'e830M.sh' to 'e830_ft.sh' if you wished to use that finetuned version."
      ],
      "metadata": {
        "id": "S9UqlcnlyfKF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "\n",
        "# Download the file\n",
        "files.download('train.txt')\n",
        "\n",
        "# ------------ ^^^^^^^^^^ Replace this to the model path."
      ],
      "metadata": {
        "id": "sSo3Tgvh6MXx"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}