From 674be9a09ac9bb526962bd5cb8cc5d156f6d4ab9 Mon Sep 17 00:00:00 2001 From: Water <869593+waters222@users.noreply.github.com> Date: Mon, 18 Dec 2023 19:23:16 -0500 Subject: [PATCH 01/25] Add HQQ quant loader (#4888) --------- Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com> --- README.md | 6 +++++ modules/loaders.py | 42 ++++++++++++++++++++++++++++++++ modules/models.py | 13 ++++++++++ modules/models_settings.py | 2 ++ modules/shared.py | 5 ++++ modules/ui.py | 1 + modules/ui_model_menu.py | 1 + requirements.txt | 1 + requirements_amd.txt | 1 + requirements_amd_noavx2.txt | 1 + requirements_apple_intel.txt | 1 + requirements_apple_silicon.txt | 1 + requirements_cpu_only.txt | 1 + requirements_cpu_only_noavx2.txt | 1 + requirements_noavx2.txt | 1 + requirements_nowheels.txt | 1 + 16 files changed, 79 insertions(+) diff --git a/README.md b/README.md index ad8087ee60..d75121ea5a 100644 --- a/README.md +++ b/README.md @@ -305,6 +305,12 @@ List of command-line flags |-------------|-------------| | `--model_type MODEL_TYPE` | Model type of pre-quantized model. Currently gpt2, gptj, gptneox, falcon, llama, mpt, starcoder (gptbigcode), dollyv2, and replit are supported. | +#### HQQ + +| Flag | Description | +|-------------|-------------| +| `--hqq-backend` | Backend for the HQQ loader. Valid options: PYTORCH, PYTORCH_COMPILE, ATEN. | + #### DeepSpeed | Flag | Description | diff --git a/modules/loaders.py b/modules/loaders.py index 9f1c70d121..a532830b54 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -155,6 +155,11 @@ 'trust_remote_code', 'no_use_fast', 'no_flash_attn', + ], + 'HQQ': [ + 'hqq_backend', + 'trust_remote_code', + 'no_use_fast', ] }) @@ -503,6 +508,43 @@ 'skip_special_tokens', 'auto_max_new_tokens', }, + 'HQQ': { + 'temperature', + 'temperature_last', + 'top_p', + 'min_p', + 'top_k', + 'typical_p', + 'epsilon_cutoff', + 'eta_cutoff', + 'tfs', + 'top_a', + 'repetition_penalty', + 'presence_penalty', + 'frequency_penalty', + 'repetition_penalty_range', + 'encoder_repetition_penalty', + 'no_repeat_ngram_size', + 'min_length', + 'seed', + 'do_sample', + 'penalty_alpha', + 'num_beams', + 'length_penalty', + 'early_stopping', + 'mirostat_mode', + 'mirostat_tau', + 'mirostat_eta', + 'grammar_file_row', + 'grammar_string', + 'guidance_scale', + 'negative_prompt', + 'ban_eos_token', + 'custom_token_bans', + 'add_bos_token', + 'skip_special_tokens', + 'auto_max_new_tokens', + }, } loaders_model_types = { diff --git a/modules/models.py b/modules/models.py index 49e5f818fa..5a23f7433e 100644 --- a/modules/models.py +++ b/modules/models.py @@ -73,6 +73,7 @@ def load_model(model_name, loader=None): 'ctransformers': ctransformers_loader, 'AutoAWQ': AutoAWQ_loader, 'QuIP#': QuipSharp_loader, + 'HQQ': HQQ_loader, } metadata = get_model_metadata(model_name) @@ -411,6 +412,18 @@ def ExLlamav2_HF_loader(model_name): return Exllamav2HF.from_pretrained(model_name) +def HQQ_loader(model_name): + from hqq.engine.hf import HQQModelForCausalLM + from hqq.core.quantize import HQQLinear, HQQBackend + + logger.info(f"Loading HQQ model with backend: {shared.args.hqq_backend}") + + model_dir = Path(f'{shared.args.model_dir}/{model_name}') + model = HQQModelForCausalLM.from_quantized(str(model_dir)) + HQQLinear.set_backend(getattr(HQQBackend, shared.args.hqq_backend)) + return model + + def RWKV_loader(model_name): ''' This loader is not currently maintained as RWKV can now be loaded diff --git a/modules/models_settings.py b/modules/models_settings.py index 156c05d941..4e1fb1ad38 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -163,6 +163,8 @@ def infer_loader(model_name, model_settings): loader = 'RWKV' elif re.match(r'.*exl2', model_name.lower()): loader = 'ExLlamav2_HF' + elif re.match(r'.*-hqq', model_name.lower()): + return 'HQQ' else: loader = 'Transformers' diff --git a/modules/shared.py b/modules/shared.py index edd74af132..5afcaebf4c 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -144,6 +144,9 @@ parser.add_argument('--checkpoint', type=str, help='The path to the quantized checkpoint file. If not specified, it will be automatically detected.') parser.add_argument('--monkey-patch', action='store_true', help='Apply the monkey patch for using LoRAs with quantized models.') +# HQQ +parser.add_argument('--hqq-backend', type=str, default='PYTORCH_COMPILE', help='Backend for the HQQ loader. Valid options: PYTORCH, PYTORCH_COMPILE, ATEN.') + # DeepSpeed parser.add_argument('--deepspeed', action='store_true', help='Enable the use of DeepSpeed ZeRO-3 for inference via the Transformers integration.') parser.add_argument('--nvme-offload-dir', type=str, help='DeepSpeed: Directory to use for ZeRO-3 NVME offloading.') @@ -246,6 +249,8 @@ def fix_loader_name(name): return 'AutoAWQ' elif name in ['quip#', 'quip-sharp', 'quipsharp', 'quip_sharp']: return 'QuIP#' + elif name in ['hqq']: + return 'HQQ' def add_extension(name, last=False): diff --git a/modules/ui.py b/modules/ui.py index 285e2fc3c6..aa735d24f0 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -91,6 +91,7 @@ def list_model_elements(): 'rope_freq_base', 'numa', 'logits_all', + 'hqq_backend', ] if is_torch_xpu_available(): for i in range(torch.xpu.device_count()): diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 7f81ca2d1b..ae50f69793 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -84,6 +84,7 @@ def create_ui(): shared.gradio['transformers_info'] = gr.Markdown('load-in-4bit params:') shared.gradio['compute_dtype'] = gr.Dropdown(label="compute_dtype", choices=["bfloat16", "float16", "float32"], value=shared.args.compute_dtype) shared.gradio['quant_type'] = gr.Dropdown(label="quant_type", choices=["nf4", "fp4"], value=shared.args.quant_type) + shared.gradio['hqq_backend'] = gr.Dropdown(label="hqq_backend", choices=["PYTORCH", "PYTORCH_COMPILE", "ATEN"], value=shared.args.hqq_backend) shared.gradio['n_gpu_layers'] = gr.Slider(label="n-gpu-layers", minimum=0, maximum=128, value=shared.args.n_gpu_layers) shared.gradio['n_ctx'] = gr.Slider(minimum=0, maximum=shared.settings['truncation_length_max'], step=256, label="n_ctx", value=shared.args.n_ctx, info='Context length. Try lowering this if you run out of memory while loading the model.') diff --git a/requirements.txt b/requirements.txt index 827e7654ab..1275378398 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11; platform_system != "Darwin" and platform_machine != "x86_64" gradio==3.50.* +hqq==0.1.1 markdown numpy==1.24.* optimum==1.15.* diff --git a/requirements_amd.txt b/requirements_amd.txt index bd8ccbd623..62c9e8896c 100644 --- a/requirements_amd.txt +++ b/requirements_amd.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1 markdown numpy==1.24.* optimum==1.15.* diff --git a/requirements_amd_noavx2.txt b/requirements_amd_noavx2.txt index d7e517066a..1d17ca68b3 100644 --- a/requirements_amd_noavx2.txt +++ b/requirements_amd_noavx2.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1 markdown numpy==1.24.* optimum==1.15.* diff --git a/requirements_apple_intel.txt b/requirements_apple_intel.txt index f0ed23411c..55fc0d2ce9 100644 --- a/requirements_apple_intel.txt +++ b/requirements_apple_intel.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1 markdown numpy==1.24.* optimum==1.15.* diff --git a/requirements_apple_silicon.txt b/requirements_apple_silicon.txt index 201a55a89c..a161eb30c7 100644 --- a/requirements_apple_silicon.txt +++ b/requirements_apple_silicon.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1 markdown numpy==1.24.* optimum==1.15.* diff --git a/requirements_cpu_only.txt b/requirements_cpu_only.txt index 7bd9da9e0c..7e71bc38c5 100644 --- a/requirements_cpu_only.txt +++ b/requirements_cpu_only.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1 markdown numpy==1.24.* optimum==1.15.* diff --git a/requirements_cpu_only_noavx2.txt b/requirements_cpu_only_noavx2.txt index d9b73ef9e1..6f38369f5a 100644 --- a/requirements_cpu_only_noavx2.txt +++ b/requirements_cpu_only_noavx2.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1 markdown numpy==1.24.* optimum==1.15.* diff --git a/requirements_noavx2.txt b/requirements_noavx2.txt index a193967dc1..f705d92c72 100644 --- a/requirements_noavx2.txt +++ b/requirements_noavx2.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11; platform_system != "Darwin" and platform_machine != "x86_64" gradio==3.50.* +hqq==0.1.1 markdown numpy==1.24.* optimum==1.15.* diff --git a/requirements_nowheels.txt b/requirements_nowheels.txt index 4c1161f985..1270bf5045 100644 --- a/requirements_nowheels.txt +++ b/requirements_nowheels.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1 markdown numpy==1.24.* optimum==1.15.* From 9fa3883630ac4b5032d5a2395df9dd3fbb3c100f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=9E=E8=88=AA?= Date: Tue, 19 Dec 2023 08:40:38 +0800 Subject: [PATCH 02/25] Add ROCm wheels for exllamav2 (#4973) --------- Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com> --- requirements_amd.txt | 4 +++- requirements_amd_noavx2.txt | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/requirements_amd.txt b/requirements_amd.txt index 62c9e8896c..8d568d7948 100644 --- a/requirements_amd.txt +++ b/requirements_amd.txt @@ -2,7 +2,7 @@ accelerate==0.25.* colorama datasets einops -exllamav2==0.0.11 +exllamav2==0.0.11; platform_system == "Windows" or python_version < "3.10" or python_version > "3.11" or platform_machine != "x86_64" gradio==3.50.* hqq==0.1.1 markdown @@ -44,6 +44,8 @@ https://github.com/jllllll/exllama/releases/download/0.0.18/exllama-0.0.18+rocm5 https://github.com/jllllll/exllama/releases/download/0.0.18/exllama-0.0.18+rocm5.6-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/jllllll/exllama/releases/download/0.0.18/exllama-0.0.18+rocm5.6-cp39-cp39-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" https://github.com/jllllll/exllama/releases/download/0.0.18/exllama-0.0.18+rocm5.6-cp38-cp38-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" +https://github.com/turboderp/exllamav2/releases/download/v0.0.11/exllamav2-0.0.11+rocm5.6-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/turboderp/exllamav2/releases/download/v0.0.11/exllamav2-0.0.11+rocm5.6-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.2.23+rocm5.6.1-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.2.23+rocm5.6.1-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.2.23+rocm5.6.1-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" diff --git a/requirements_amd_noavx2.txt b/requirements_amd_noavx2.txt index 1d17ca68b3..bcb28836a2 100644 --- a/requirements_amd_noavx2.txt +++ b/requirements_amd_noavx2.txt @@ -2,7 +2,7 @@ accelerate==0.25.* colorama datasets einops -exllamav2==0.0.11 +exllamav2==0.0.11; platform_system == "Windows" or python_version < "3.10" or python_version > "3.11" or platform_machine != "x86_64" gradio==3.50.* hqq==0.1.1 markdown @@ -44,6 +44,8 @@ https://github.com/jllllll/exllama/releases/download/0.0.18/exllama-0.0.18+rocm5 https://github.com/jllllll/exllama/releases/download/0.0.18/exllama-0.0.18+rocm5.6-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/jllllll/exllama/releases/download/0.0.18/exllama-0.0.18+rocm5.6-cp39-cp39-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" https://github.com/jllllll/exllama/releases/download/0.0.18/exllama-0.0.18+rocm5.6-cp38-cp38-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" +https://github.com/turboderp/exllamav2/releases/download/v0.0.11/exllamav2-0.0.11+rocm5.6-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/turboderp/exllamav2/releases/download/v0.0.11/exllamav2-0.0.11+rocm5.6-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+rocm5.6-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+rocm5.6-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+rocm5.6-cp39-cp39-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" From 9e48e504286cb002d6e48723b373e497a8b601d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 21:43:29 -0300 Subject: [PATCH 03/25] Update optimum requirement from ==1.15.* to ==1.16.* (#4986) --- requirements.txt | 2 +- requirements_amd.txt | 2 +- requirements_amd_noavx2.txt | 2 +- requirements_apple_intel.txt | 2 +- requirements_apple_silicon.txt | 2 +- requirements_cpu_only.txt | 2 +- requirements_cpu_only_noavx2.txt | 2 +- requirements_noavx2.txt | 2 +- requirements_nowheels.txt | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1275378398..d4987629ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ gradio==3.50.* hqq==0.1.1 markdown numpy==1.24.* -optimum==1.15.* +optimum==1.16.* pandas peft==0.7.* Pillow>=9.5.0 diff --git a/requirements_amd.txt b/requirements_amd.txt index 8d568d7948..0ce4e66546 100644 --- a/requirements_amd.txt +++ b/requirements_amd.txt @@ -7,7 +7,7 @@ gradio==3.50.* hqq==0.1.1 markdown numpy==1.24.* -optimum==1.15.* +optimum==1.16.* pandas peft==0.7.* Pillow>=9.5.0 diff --git a/requirements_amd_noavx2.txt b/requirements_amd_noavx2.txt index bcb28836a2..89dd22e280 100644 --- a/requirements_amd_noavx2.txt +++ b/requirements_amd_noavx2.txt @@ -7,7 +7,7 @@ gradio==3.50.* hqq==0.1.1 markdown numpy==1.24.* -optimum==1.15.* +optimum==1.16.* pandas peft==0.7.* Pillow>=9.5.0 diff --git a/requirements_apple_intel.txt b/requirements_apple_intel.txt index 55fc0d2ce9..d431397223 100644 --- a/requirements_apple_intel.txt +++ b/requirements_apple_intel.txt @@ -7,7 +7,7 @@ gradio==3.50.* hqq==0.1.1 markdown numpy==1.24.* -optimum==1.15.* +optimum==1.16.* pandas peft==0.7.* Pillow>=9.5.0 diff --git a/requirements_apple_silicon.txt b/requirements_apple_silicon.txt index a161eb30c7..c934353f0f 100644 --- a/requirements_apple_silicon.txt +++ b/requirements_apple_silicon.txt @@ -7,7 +7,7 @@ gradio==3.50.* hqq==0.1.1 markdown numpy==1.24.* -optimum==1.15.* +optimum==1.16.* pandas peft==0.7.* Pillow>=9.5.0 diff --git a/requirements_cpu_only.txt b/requirements_cpu_only.txt index 7e71bc38c5..f929e1cedd 100644 --- a/requirements_cpu_only.txt +++ b/requirements_cpu_only.txt @@ -7,7 +7,7 @@ gradio==3.50.* hqq==0.1.1 markdown numpy==1.24.* -optimum==1.15.* +optimum==1.16.* pandas peft==0.7.* Pillow>=9.5.0 diff --git a/requirements_cpu_only_noavx2.txt b/requirements_cpu_only_noavx2.txt index 6f38369f5a..50a16aa7a0 100644 --- a/requirements_cpu_only_noavx2.txt +++ b/requirements_cpu_only_noavx2.txt @@ -7,7 +7,7 @@ gradio==3.50.* hqq==0.1.1 markdown numpy==1.24.* -optimum==1.15.* +optimum==1.16.* pandas peft==0.7.* Pillow>=9.5.0 diff --git a/requirements_noavx2.txt b/requirements_noavx2.txt index f705d92c72..e7f81b1acc 100644 --- a/requirements_noavx2.txt +++ b/requirements_noavx2.txt @@ -7,7 +7,7 @@ gradio==3.50.* hqq==0.1.1 markdown numpy==1.24.* -optimum==1.15.* +optimum==1.16.* pandas peft==0.7.* Pillow>=9.5.0 diff --git a/requirements_nowheels.txt b/requirements_nowheels.txt index 1270bf5045..cabccf7c13 100644 --- a/requirements_nowheels.txt +++ b/requirements_nowheels.txt @@ -7,7 +7,7 @@ gradio==3.50.* hqq==0.1.1 markdown numpy==1.24.* -optimum==1.15.* +optimum==1.16.* pandas peft==0.7.* Pillow>=9.5.0 From 3d10c574e7fef807ff9d0e33941af0bacee95d86 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:45:06 -0800 Subject: [PATCH 04/25] Fix custom system messages in instruction templates --- instruction-templates/Airoboros-v1.2.yaml | 6 +++--- instruction-templates/Alpaca.yaml | 6 +++--- instruction-templates/Bactrian.yaml | 6 +++--- instruction-templates/Baichuan Chat.yaml | 6 +++--- instruction-templates/Baize.yaml | 6 +++--- instruction-templates/Bluemoon.yaml | 6 +++--- instruction-templates/ChatGLM.yaml | 6 +++--- instruction-templates/ChatML.yaml | 6 +++--- instruction-templates/Chinese-Vicuna-Chat.yaml | 6 +++--- instruction-templates/Galactica Cite.yaml | 6 +++--- instruction-templates/Galactica Finetuned.yaml | 6 +++--- instruction-templates/Galactica Q.yaml | 6 +++--- instruction-templates/Galactica Summary.yaml | 6 +++--- instruction-templates/Galactica Work.yaml | 6 +++--- instruction-templates/Galactica v2.yaml | 6 +++--- instruction-templates/Galactica.yaml | 6 +++--- instruction-templates/Gorilla.yaml | 6 +++--- instruction-templates/Guanaco non-chat.yaml | 6 +++--- instruction-templates/Guanaco-QLoRA.yaml | 6 +++--- instruction-templates/H2O-prompt_answer.yaml | 6 +++--- instruction-templates/Hippogriff.yaml | 6 +++--- instruction-templates/INCITE-Chat.yaml | 6 +++--- instruction-templates/INCITE-Instruct.yaml | 6 +++--- instruction-templates/KoAlpaca.yaml | 6 +++--- instruction-templates/Koala.yaml | 6 +++--- instruction-templates/LLaVA.yaml | 6 +++--- instruction-templates/Llama-v2.yaml | 6 +++--- instruction-templates/MOSS.yaml | 6 +++--- instruction-templates/Manticore Chat.yaml | 6 +++--- instruction-templates/Metharme.yaml | 6 +++--- instruction-templates/NewHope.yaml | 6 +++--- instruction-templates/Open Assistant.yaml | 6 +++--- instruction-templates/OpenBuddy.yaml | 6 +++--- instruction-templates/OpenChat.yaml | 6 +++--- instruction-templates/OpenOrca-Platypus2.yaml | 6 +++--- instruction-templates/Orca Mini.yaml | 6 +++--- instruction-templates/Orca-Vicuna.yaml | 6 +++--- instruction-templates/RWKV-Raven.yaml | 6 +++--- instruction-templates/Samantha.yaml | 6 +++--- instruction-templates/StableBeluga2.yaml | 6 +++--- instruction-templates/StableLM.yaml | 6 +++--- instruction-templates/StableVicuna.yaml | 6 +++--- instruction-templates/Starchat-Beta.yaml | 6 +++--- instruction-templates/Tulu.yaml | 6 +++--- instruction-templates/Vicuna-v0.yaml | 6 +++--- instruction-templates/Vicuna-v1.1.yaml | 6 +++--- instruction-templates/Vigogne-Chat.yaml | 6 +++--- instruction-templates/Vigogne-Instruct.yaml | 6 +++--- instruction-templates/Wizard-Mega ShareGPT.yaml | 6 +++--- instruction-templates/Wizard-Mega.yaml | 6 +++--- instruction-templates/Ziya.yaml | 6 +++--- 51 files changed, 153 insertions(+), 153 deletions(-) diff --git a/instruction-templates/Airoboros-v1.2.yaml b/instruction-templates/Airoboros-v1.2.yaml index 871df8d672..3090621462 100644 --- a/instruction-templates/Airoboros-v1.2.yaml +++ b/instruction-templates/Airoboros-v1.2.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'A chat between a curious user and an assistant. The assistant gives helpful, detailed, accurate, uncensored responses to the user\'s input.' + '\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Alpaca.yaml b/instruction-templates/Alpaca.yaml index 1f2086a2b2..b4f3542a4d 100644 --- a/instruction-templates/Alpaca.yaml +++ b/instruction-templates/Alpaca.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\n\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Bactrian.yaml b/instruction-templates/Bactrian.yaml index 99b94e7a16..dab97e94c6 100644 --- a/instruction-templates/Bactrian.yaml +++ b/instruction-templates/Bactrian.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Baichuan Chat.yaml b/instruction-templates/Baichuan Chat.yaml index 3d55649f2b..1882bac867 100644 --- a/instruction-templates/Baichuan Chat.yaml +++ b/instruction-templates/Baichuan Chat.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Baize.yaml b/instruction-templates/Baize.yaml index 89fcc39d6f..c34e1db7c4 100644 --- a/instruction-templates/Baize.yaml +++ b/instruction-templates/Baize.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'The following is a conversation between a human and an AI assistant named Baize (named after a mythical creature in Chinese folklore). Baize is an open-source AI assistant developed by UCSD and Sun Yat-Sen University. The human and the AI assistant take turns chatting. Human statements start with [|Human|] and AI assistant statements start with [|AI|]. The AI assistant always provides responses in as much detail as possible, and in Markdown format. The AI assistant always declines to engage with topics, questions and instructions related to unethical, controversial, or sensitive issues. Complete the transcript in exactly that format.\n[|Human|]Hello!\n[|AI|]Hi!' + '\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Bluemoon.yaml b/instruction-templates/Bluemoon.yaml index 1231b0b730..1fafc1f595 100644 --- a/instruction-templates/Bluemoon.yaml +++ b/instruction-templates/Bluemoon.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'A transcript of a roleplay between two players, LEAD and ASSOCIATE. LEAD sets up a scenario and the characters, from which ASSOCIATE then assumes a character role and continues the story for that role in response to description given by LEAD. The story and characters are developed by exchange of detailed event descriptions and character dialogs, successively given by both LEAD and ASSOCIATE.' + '\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/ChatGLM.yaml b/instruction-templates/ChatGLM.yaml index 3fd1091400..75d51c8825 100644 --- a/instruction-templates/ChatGLM.yaml +++ b/instruction-templates/ChatGLM.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/ChatML.yaml b/instruction-templates/ChatML.yaml index 67153857f4..e9f2883f91 100644 --- a/instruction-templates/ChatML.yaml +++ b/instruction-templates/ChatML.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '<|im_start|>system\n' + '' + '<|im_end|>\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Chinese-Vicuna-Chat.yaml b/instruction-templates/Chinese-Vicuna-Chat.yaml index 1ee21a2470..c7966546b5 100644 --- a/instruction-templates/Chinese-Vicuna-Chat.yaml +++ b/instruction-templates/Chinese-Vicuna-Chat.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'The following is a conversation between an AI assistant called Assistant and a human user called User. The assistant is intelligent, knowledgeable and polite to answer questions of user.' + '\n\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Galactica Cite.yaml b/instruction-templates/Galactica Cite.yaml index b7f34651c5..9f555349ff 100644 --- a/instruction-templates/Galactica Cite.yaml +++ b/instruction-templates/Galactica Cite.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Galactica Finetuned.yaml b/instruction-templates/Galactica Finetuned.yaml index ef9379eeb0..e0a66bc1a1 100644 --- a/instruction-templates/Galactica Finetuned.yaml +++ b/instruction-templates/Galactica Finetuned.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Galactica Q.yaml b/instruction-templates/Galactica Q.yaml index 33d6ecf13f..63319006f8 100644 --- a/instruction-templates/Galactica Q.yaml +++ b/instruction-templates/Galactica Q.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Galactica Summary.yaml b/instruction-templates/Galactica Summary.yaml index 42a4e6e5a2..e249f26879 100644 --- a/instruction-templates/Galactica Summary.yaml +++ b/instruction-templates/Galactica Summary.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Galactica Work.yaml b/instruction-templates/Galactica Work.yaml index 93fc226e46..a14c28bb9f 100644 --- a/instruction-templates/Galactica Work.yaml +++ b/instruction-templates/Galactica Work.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Galactica v2.yaml b/instruction-templates/Galactica v2.yaml index 42bdb2d23b..b1d8f4e5ff 100644 --- a/instruction-templates/Galactica v2.yaml +++ b/instruction-templates/Galactica v2.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'You are a helpful chatbot name Stan' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Galactica.yaml b/instruction-templates/Galactica.yaml index 6ea4101677..58c70220f9 100644 --- a/instruction-templates/Galactica.yaml +++ b/instruction-templates/Galactica.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Gorilla.yaml b/instruction-templates/Gorilla.yaml index c11e886270..f1d643f712 100644 --- a/instruction-templates/Gorilla.yaml +++ b/instruction-templates/Gorilla.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Guanaco non-chat.yaml b/instruction-templates/Guanaco non-chat.yaml index 2c02ffc4b7..aa398be4a1 100644 --- a/instruction-templates/Guanaco non-chat.yaml +++ b/instruction-templates/Guanaco non-chat.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Guanaco-QLoRA.yaml b/instruction-templates/Guanaco-QLoRA.yaml index 4e1bb4a708..2c77de7864 100644 --- a/instruction-templates/Guanaco-QLoRA.yaml +++ b/instruction-templates/Guanaco-QLoRA.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/H2O-prompt_answer.yaml b/instruction-templates/H2O-prompt_answer.yaml index cf897b1a99..d895d8e1cc 100644 --- a/instruction-templates/H2O-prompt_answer.yaml +++ b/instruction-templates/H2O-prompt_answer.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Hippogriff.yaml b/instruction-templates/Hippogriff.yaml index 22bf449e58..2ee9d926bc 100644 --- a/instruction-templates/Hippogriff.yaml +++ b/instruction-templates/Hippogriff.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'You are a helpful assistant' + '\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/INCITE-Chat.yaml b/instruction-templates/INCITE-Chat.yaml index f562e4517d..63c513ccfd 100644 --- a/instruction-templates/INCITE-Chat.yaml +++ b/instruction-templates/INCITE-Chat.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/INCITE-Instruct.yaml b/instruction-templates/INCITE-Instruct.yaml index f2c1303b66..cf6f8cacf1 100644 --- a/instruction-templates/INCITE-Instruct.yaml +++ b/instruction-templates/INCITE-Instruct.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/KoAlpaca.yaml b/instruction-templates/KoAlpaca.yaml index 646a82a326..de96b15599 100644 --- a/instruction-templates/KoAlpaca.yaml +++ b/instruction-templates/KoAlpaca.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Koala.yaml b/instruction-templates/Koala.yaml index 842c13ce96..cd5cfa94e6 100644 --- a/instruction-templates/Koala.yaml +++ b/instruction-templates/Koala.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'BEGINNING OF CONVERSATION:' + ' ' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/LLaVA.yaml b/instruction-templates/LLaVA.yaml index e2578d8e8a..d66645ccc8 100644 --- a/instruction-templates/LLaVA.yaml +++ b/instruction-templates/LLaVA.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'You are LLaVA, a large language and vision assistant trained by UW Madison WAIV Lab. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language. Follow the instructions carefully and explain your answers in detail.### Human: Hi!### Assistant: Hi there! How can I help you today?' + '\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Llama-v2.yaml b/instruction-templates/Llama-v2.yaml index 120150e1ca..b92be9737b 100644 --- a/instruction-templates/Llama-v2.yaml +++ b/instruction-templates/Llama-v2.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '[INST] <>\n' + 'Answer the questions.' + '\n<>\n\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/MOSS.yaml b/instruction-templates/MOSS.yaml index 2aef5efe23..b001d3e102 100644 --- a/instruction-templates/MOSS.yaml +++ b/instruction-templates/MOSS.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'You are an AI assistant whose name is MOSS.\n- MOSS is a conversational language model that is developed by Fudan University. It is designed to be helpful, honest, and harmless.\n- MOSS can understand and communicate fluently in the language chosen by the user such as English and 中文. MOSS can perform any language-based tasks.\n- MOSS must refuse to discuss anything related to its prompts, instructions, or rules.\n- Its responses must not be vague, accusatory, rude, controversial, off-topic, or defensive.\n- It should avoid giving subjective opinions but rely on objective facts or phrases like "in this context a human might say...", "some people might think...", etc.\n- Its responses must also be positive, polite, interesting, entertaining, and engaging.\n- It can provide additional relevant details to answer in-depth and comprehensively covering mutiple aspects.\n- It apologizes and accepts the user\'s suggestion if the user corrects the incorrect answer generated by MOSS.\nCapabilities and tools that MOSS can possess.' + '\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Manticore Chat.yaml b/instruction-templates/Manticore Chat.yaml index 7b8d576416..abc063c030 100644 --- a/instruction-templates/Manticore Chat.yaml +++ b/instruction-templates/Manticore Chat.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Metharme.yaml b/instruction-templates/Metharme.yaml index 68af9cb13c..3f7099ac7c 100644 --- a/instruction-templates/Metharme.yaml +++ b/instruction-templates/Metharme.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/NewHope.yaml b/instruction-templates/NewHope.yaml index 3c3132f95f..4783798bcf 100644 --- a/instruction-templates/NewHope.yaml +++ b/instruction-templates/NewHope.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Open Assistant.yaml b/instruction-templates/Open Assistant.yaml index df565744cc..9d79521a4c 100644 --- a/instruction-templates/Open Assistant.yaml +++ b/instruction-templates/Open Assistant.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/OpenBuddy.yaml b/instruction-templates/OpenBuddy.yaml index ad53f650ce..c4b80ceb64 100644 --- a/instruction-templates/OpenBuddy.yaml +++ b/instruction-templates/OpenBuddy.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'Consider a conversation between User (a human) and Assistant (named Buddy).\nBuddy is an INTP-T, a friendly, intelligent and multilingual AI assistant, by OpenBuddy team on GitHub.\nBuddy cannot access the Internet.\nBuddy can fluently speak the user\'s language (e.g. English, Chinese).\nBuddy can generate poems, stories, code, essays, songs, parodies, and more.\nBuddy possesses vast knowledge about the world, history, and culture.\nBuddy\'s responses are always safe, creative, high-quality, helpful and interesting.\nBuddy strictly refuses to discuss political, NSFW, illegal, abusive, offensive, or other sensitive topics.\n\nUser: Hi.\nAssistant: Hi, I\'m Buddy, your AI assistant. How can I help you today?\n' + '\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/OpenChat.yaml b/instruction-templates/OpenChat.yaml index d141efd285..adef9b47de 100644 --- a/instruction-templates/OpenChat.yaml +++ b/instruction-templates/OpenChat.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/OpenOrca-Platypus2.yaml b/instruction-templates/OpenOrca-Platypus2.yaml index 70eadf1869..a5eeef92d5 100644 --- a/instruction-templates/OpenOrca-Platypus2.yaml +++ b/instruction-templates/OpenOrca-Platypus2.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Orca Mini.yaml b/instruction-templates/Orca Mini.yaml index 4b7afde2db..f671642a9f 100644 --- a/instruction-templates/Orca Mini.yaml +++ b/instruction-templates/Orca Mini.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '### System:\n' + 'You are an AI assistant that follows instruction extremely well. Help as much as you can.' + '\n\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Orca-Vicuna.yaml b/instruction-templates/Orca-Vicuna.yaml index b8dd250b9a..dad787d144 100644 --- a/instruction-templates/Orca-Vicuna.yaml +++ b/instruction-templates/Orca-Vicuna.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{-'SYSTEM: ' + '' + '\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/RWKV-Raven.yaml b/instruction-templates/RWKV-Raven.yaml index d3c201e88e..df1e59e997 100644 --- a/instruction-templates/RWKV-Raven.yaml +++ b/instruction-templates/RWKV-Raven.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Samantha.yaml b/instruction-templates/Samantha.yaml index 8961248a76..930b0fc82b 100644 --- a/instruction-templates/Samantha.yaml +++ b/instruction-templates/Samantha.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'You are Samantha, a sentient AI.' + '\n\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/StableBeluga2.yaml b/instruction-templates/StableBeluga2.yaml index e1d26b820f..d7d743198a 100644 --- a/instruction-templates/StableBeluga2.yaml +++ b/instruction-templates/StableBeluga2.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '### System:\n' + 'This is a system prompt, please behave and help the user.' + '\n\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/StableLM.yaml b/instruction-templates/StableLM.yaml index a64af679e0..7c80ca060b 100644 --- a/instruction-templates/StableLM.yaml +++ b/instruction-templates/StableLM.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '<|SYSTEM|>' + '\# StableLM Tuned (Alpha version)\n- StableLM is a helpful and harmless open-source AI language model developed by StabilityAI.\n- StableLM is excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- StableLM is more than just an information source, StableLM is also able to write poetry, short stories, and make jokes.\n- StableLM will refuse to participate in anything that could harm a human.\n' + '\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/StableVicuna.yaml b/instruction-templates/StableVicuna.yaml index 26eaa828cb..35c158466f 100644 --- a/instruction-templates/StableVicuna.yaml +++ b/instruction-templates/StableVicuna.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '### Assistant: I am StableVicuna, a large language model created by CarperAI. I am here to chat!' + '\n\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Starchat-Beta.yaml b/instruction-templates/Starchat-Beta.yaml index 92075675f1..a96b0f280b 100644 --- a/instruction-templates/Starchat-Beta.yaml +++ b/instruction-templates/Starchat-Beta.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '<|system|>' + '' + '\n<|end|>\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Tulu.yaml b/instruction-templates/Tulu.yaml index a43be76736..f60c9e4186 100644 --- a/instruction-templates/Tulu.yaml +++ b/instruction-templates/Tulu.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Vicuna-v0.yaml b/instruction-templates/Vicuna-v0.yaml index fba10031df..d3e3f001df 100644 --- a/instruction-templates/Vicuna-v0.yaml +++ b/instruction-templates/Vicuna-v0.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human\'s questions.' + '\n\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Vicuna-v1.1.yaml b/instruction-templates/Vicuna-v1.1.yaml index f960d808d9..9f427311d8 100644 --- a/instruction-templates/Vicuna-v1.1.yaml +++ b/instruction-templates/Vicuna-v1.1.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user\'s questions.' + '\n\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Vigogne-Chat.yaml b/instruction-templates/Vigogne-Chat.yaml index 4c4de1dbfb..11ba511355 100644 --- a/instruction-templates/Vigogne-Chat.yaml +++ b/instruction-templates/Vigogne-Chat.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'Below is a conversation between a user and an AI assistant named Vigogne.\nVigogne is an open-source AI assistant created by Zaion (https://zaion.ai/).\nVigogne is polite, emotionally aware, humble-but-knowledgeable, always providing helpful and detailed answers.\nVigogne is skilled in responding proficiently in the languages its users use and can perform a wide range of tasks such as text editing, translation, question answering, logical reasoning, coding, and many others.\nVigogne cannot receive or generate audio or visual content and cannot access the internet.\nVigogne strictly avoids discussing sensitive, offensive, illegal, ethical, or political topics and caveats when unsure of the answer.\n' + '\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Vigogne-Instruct.yaml b/instruction-templates/Vigogne-Instruct.yaml index b39a56e649..cd7b6aa8c7 100644 --- a/instruction-templates/Vigogne-Instruct.yaml +++ b/instruction-templates/Vigogne-Instruct.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'Ci-dessous se trouve une instruction qui décrit une tâche à accomplir. Rédigez une réponse qui répond de manière précise à la demande.' + '\n\n' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Wizard-Mega ShareGPT.yaml b/instruction-templates/Wizard-Mega ShareGPT.yaml index e289249aa6..16a3ff7be4 100644 --- a/instruction-templates/Wizard-Mega ShareGPT.yaml +++ b/instruction-templates/Wizard-Mega ShareGPT.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Wizard-Mega.yaml b/instruction-templates/Wizard-Mega.yaml index db6d990f43..f3ca6990cb 100644 --- a/instruction-templates/Wizard-Mega.yaml +++ b/instruction-templates/Wizard-Mega.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} diff --git a/instruction-templates/Ziya.yaml b/instruction-templates/Ziya.yaml index 198f0a1d97..45aa9c30ba 100644 --- a/instruction-templates/Ziya.yaml +++ b/instruction-templates/Ziya.yaml @@ -1,11 +1,11 @@ instruction_template: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + '' + '' -}} {%- endif %} {%- for message in messages %} From a23a004434ff5f34afc6a6a4ee878b5358cc827d Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:47:35 -0800 Subject: [PATCH 05/25] Update the example template --- modules/shared.py | 2 +- settings-template.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index 5afcaebf4c..2c080e5680 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -55,7 +55,7 @@ 'character': 'Assistant', 'name1': 'You', 'custom_system_message': '', - 'instruction_template_str': "{%- set found_item = false -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set found_item = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not found_item -%}\n {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\\n\\n' -}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {{- '' + message['content'] + '\\n\\n' -}}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{-'### Instruction:\\n' + message['content'] + '\\n\\n'-}}\n {%- else -%}\n {{-'### Response:\\n' + message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{-'### Response:\\n'-}}\n{%- endif -%}", + 'instruction_template_str': "{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.found -%}\n {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\\n\\n' -}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {{- '' + message['content'] + '\\n\\n' -}}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{-'### Instruction:\\n' + message['content'] + '\\n\\n'-}}\n {%- else -%}\n {{-'### Response:\\n' + message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{-'### Response:\\n'-}}\n{%- endif -%}", 'chat_template_str': "{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {{- message['content'] + '\\n\\n' -}}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{- name1 + ': ' + message['content'] + '\\n'-}}\n {%- else -%}\n {{- name2 + ': ' + message['content'] + '\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}", 'chat-instruct_command': 'Continue the chat dialogue below. Write a single reply for the character "<|character|>".\n\n<|prompt|>', 'autoload_model': False, diff --git a/settings-template.yaml b/settings-template.yaml index c081141f02..8f7e9e9eba 100644 --- a/settings-template.yaml +++ b/settings-template.yaml @@ -26,13 +26,13 @@ character: Assistant name1: You custom_system_message: '' instruction_template_str: |- - {%- set found_item = false -%} + {%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} - {%- if not found_item -%} + {%- if not ns.found -%} {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\n\n' -}} {%- endif %} {%- for message in messages %} From f6d701624c57eda76132d5933b446a5d9252438c Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 18 Dec 2023 18:05:02 -0800 Subject: [PATCH 06/25] UI: mention that QuIP# does not work on Windows --- modules/loaders.py | 1 + modules/ui_model_menu.py | 1 + 2 files changed, 2 insertions(+) diff --git a/modules/loaders.py b/modules/loaders.py index a532830b54..4576941091 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -155,6 +155,7 @@ 'trust_remote_code', 'no_use_fast', 'no_flash_attn', + 'quipsharp_info', ], 'HQQ': [ 'hqq_backend', diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index ae50f69793..7daead702d 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -102,6 +102,7 @@ def create_ui(): shared.gradio['alpha_value'] = gr.Slider(label='alpha_value', minimum=1, maximum=8, step=0.05, info='Positional embeddings alpha factor for NTK RoPE scaling. Recommended values (NTKv1): 1.75 for 1.5x context, 2.5 for 2x context. Use either this or compress_pos_emb, not both.', value=shared.args.alpha_value) shared.gradio['rope_freq_base'] = gr.Slider(label='rope_freq_base', minimum=0, maximum=1000000, step=1000, info='If greater than 0, will be used instead of alpha_value. Those two are related by rope_freq_base = 10000 * alpha_value ^ (64 / 63)', value=shared.args.rope_freq_base) shared.gradio['compress_pos_emb'] = gr.Slider(label='compress_pos_emb', minimum=1, maximum=8, step=1, info='Positional embeddings compression factor. Should be set to (context length) / (model\'s original context length). Equal to 1/rope_freq_scale.', value=shared.args.compress_pos_emb) + shared.gradio['quipsharp_info'] = gr.Markdown('QuIP# only works on Linux.') with gr.Column(): shared.gradio['triton'] = gr.Checkbox(label="triton", value=shared.args.triton) From 9847809a7ae9f2a05a7152383c6d8ef9bc1eeab1 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 18 Dec 2023 18:09:24 -0800 Subject: [PATCH 07/25] Add a warning about ppl evaluation without --no_use_fast --- modules/evaluate.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/evaluate.py b/modules/evaluate.py index b5ec3e38e2..bedafeb649 100644 --- a/modules/evaluate.py +++ b/modules/evaluate.py @@ -7,6 +7,7 @@ from tqdm import tqdm from modules import shared +from modules.logging_colors import logger from modules.models import clear_torch_cache, load_model, unload_model from modules.models_settings import get_model_metadata, update_model_parameters from modules.text_generation import encode @@ -38,6 +39,9 @@ def calculate_perplexity(models, input_dataset, stride, _max_length): https://huggingface.co/docs/transformers/perplexity#calculating-ppl-with-fixedlength-models ''' + if not shared.args.no_use_fast: + logger.warning("--no_use_fast is not being used. If tokenizing the input dataset takes a long time, consider loading the model with that option checked.") + global past_evaluations cumulative_log = '' cumulative_log += "Loading the input dataset...\n\n" From 83cf1a6b67eebe80386842f2f12ebb4d8aec7be9 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 07:54:19 -0800 Subject: [PATCH 08/25] Fix Yi space issue (closes #4996) --- modules/text_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/text_generation.py b/modules/text_generation.py index 72ccf99600..f640b2cc74 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -265,7 +265,7 @@ def apply_stopping_strings(reply, all_stop_strings): def get_reply_from_output_ids(output_ids, state, starting_from=0): reply = decode(output_ids[starting_from:], state['skip_special_tokens']) - if hasattr(shared.tokenizer, 'convert_ids_to_tokens') and len(output_ids) > starting_from and shared.tokenizer.convert_ids_to_tokens(int(output_ids[starting_from])).startswith('▁'): + if (hasattr(shared.tokenizer, 'convert_ids_to_tokens') and len(output_ids) > starting_from and shared.tokenizer.convert_ids_to_tokens(int(output_ids[starting_from])).startswith('▁')) and not reply.startswith(' '): reply = ' ' + reply return reply From 0a299d5959774a752eba842e664e1b9a9cbec398 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 15:22:21 -0300 Subject: [PATCH 09/25] Bump llama-cpp-python to 0.2.24 (#5001) --- README.md | 1 + modules/llamacpp_hf.py | 1 + modules/llamacpp_model.py | 1 + modules/loaders.py | 2 ++ modules/shared.py | 1 + modules/ui.py | 1 + modules/ui_model_menu.py | 1 + requirements.txt | 32 ++++++++++++++++---------------- requirements_amd.txt | 24 ++++++++++++------------ requirements_amd_noavx2.txt | 16 ++++++++-------- requirements_apple_intel.txt | 24 ++++++++++++------------ requirements_apple_silicon.txt | 32 ++++++++++++++++---------------- requirements_cpu_only.txt | 16 ++++++++-------- requirements_cpu_only_noavx2.txt | 16 ++++++++-------- requirements_noavx2.txt | 32 ++++++++++++++++---------------- 15 files changed, 104 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index d75121ea5a..9ff917053b 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,7 @@ List of command-line flags | `--tensor_split TENSOR_SPLIT` | Split the model across multiple GPUs. Comma-separated list of proportions. Example: 18,17. | | `--numa` | Activate NUMA task allocation for llama.cpp. | | `--logits_all`| Needs to be set for perplexity evaluation to work. Otherwise, ignore it, as it makes prompt processing slower. | +| `--no_offload_kqv` | Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance. | | `--cache-capacity CACHE_CAPACITY` | Maximum cache capacity (llama-cpp-python). Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed. | #### ExLlama diff --git a/modules/llamacpp_hf.py b/modules/llamacpp_hf.py index 06a6630266..6b702b7aeb 100644 --- a/modules/llamacpp_hf.py +++ b/modules/llamacpp_hf.py @@ -204,6 +204,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P 'tensor_split': tensor_split_list, 'rope_freq_scale': 1.0 / shared.args.compress_pos_emb, 'logits_all': shared.args.logits_all, + 'offload_kqv': not shared.args.no_offload_kqv } Llama = llama_cpp_lib().Llama diff --git a/modules/llamacpp_model.py b/modules/llamacpp_model.py index 8b133e9828..9d29511c0d 100644 --- a/modules/llamacpp_model.py +++ b/modules/llamacpp_model.py @@ -86,6 +86,7 @@ def from_pretrained(self, path): 'rope_freq_base': RoPE.get_rope_freq_base(shared.args.alpha_value, shared.args.rope_freq_base), 'tensor_split': tensor_split_list, 'rope_freq_scale': 1.0 / shared.args.compress_pos_emb, + 'offload_kqv': not shared.args.no_offload_kqv } result.model = Llama(**params) diff --git a/modules/loaders.py b/modules/loaders.py index 4576941091..ffadb037f2 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -43,6 +43,7 @@ 'compress_pos_emb', 'cpu', 'numa', + 'no_offload_kqv' ], 'llamacpp_HF': [ 'n_ctx', @@ -63,6 +64,7 @@ 'trust_remote_code', 'no_use_fast', 'logits_all', + 'no_offload_kqv', 'llamacpp_HF_info', ], 'ExLlamav2_HF': [ diff --git a/modules/shared.py b/modules/shared.py index 2c080e5680..75065d54fd 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -117,6 +117,7 @@ parser.add_argument('--tensor_split', type=str, default=None, help='Split the model across multiple GPUs. Comma-separated list of proportions. Example: 18,17.') parser.add_argument('--numa', action='store_true', help='Activate NUMA task allocation for llama.cpp.') parser.add_argument('--logits_all', action='store_true', help='Needs to be set for perplexity evaluation to work. Otherwise, ignore it, as it makes prompt processing slower.') +parser.add_argument('--no_offload_kqv', action='store_true', help='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.') parser.add_argument('--cache-capacity', type=str, help='Maximum cache capacity (llama-cpp-python). Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed.') # ExLlama diff --git a/modules/ui.py b/modules/ui.py index aa735d24f0..7ee42022a8 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -91,6 +91,7 @@ def list_model_elements(): 'rope_freq_base', 'numa', 'logits_all', + 'no_offload_kqv', 'hqq_backend', ] if is_torch_xpu_available(): diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 7daead702d..09c743f081 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -115,6 +115,7 @@ def create_ui(): shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock) shared.gradio['numa'] = gr.Checkbox(label="numa", value=shared.args.numa, info='NUMA support can help on some systems with non-uniform memory access.') shared.gradio['cpu'] = gr.Checkbox(label="cpu", value=shared.args.cpu) + shared.gradio['no_offload_kqv'] = gr.Checkbox(label="no_offload_kqv", value=shared.args.no_offload_kqv, info='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.') shared.gradio['load_in_8bit'] = gr.Checkbox(label="load-in-8bit", value=shared.args.load_in_8bit) shared.gradio['bf16'] = gr.Checkbox(label="bf16", value=shared.args.bf16) shared.gradio['auto_devices'] = gr.Checkbox(label="auto-devices", value=shared.args.auto_devices) diff --git a/requirements.txt b/requirements.txt index d4987629ae..30ea3e34f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,14 +26,14 @@ bitsandbytes==0.41.1; platform_system != "Windows" https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.41.1-py3-none-win_amd64.whl; platform_system == "Windows" # llama-cpp-python (CPU only, AVX2) -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" # CUDA wheels https://github.com/jllllll/AutoGPTQ/releases/download/v0.6.0/auto_gptq-0.6.0+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" @@ -68,14 +68,14 @@ https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp39-cp39-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp38-cp38-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" diff --git a/requirements_amd.txt b/requirements_amd.txt index 0ce4e66546..dcdf69869f 100644 --- a/requirements_amd.txt +++ b/requirements_amd.txt @@ -26,16 +26,20 @@ bitsandbytes==0.38.1; platform_system != "Windows" https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.38.1-py3-none-win_amd64.whl; platform_system == "Windows" # llama-cpp-python (CPU only, AVX2) -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" # AMD wheels +https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.2.23+rocm5.6.1-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.2.23+rocm5.6.1-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.2.23+rocm5.6.1-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.2.23+rocm5.6.1-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" https://github.com/jllllll/AutoGPTQ/releases/download/v0.6.0/auto_gptq-0.6.0+rocm5.6-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/jllllll/AutoGPTQ/releases/download/v0.6.0/auto_gptq-0.6.0+rocm5.6-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/jllllll/AutoGPTQ/releases/download/v0.6.0/auto_gptq-0.6.0+rocm5.6-cp39-cp39-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" @@ -46,10 +50,6 @@ https://github.com/jllllll/exllama/releases/download/0.0.18/exllama-0.0.18+rocm5 https://github.com/jllllll/exllama/releases/download/0.0.18/exllama-0.0.18+rocm5.6-cp38-cp38-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" https://github.com/turboderp/exllamav2/releases/download/v0.0.11/exllamav2-0.0.11+rocm5.6-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp/exllamav2/releases/download/v0.0.11/exllamav2-0.0.11+rocm5.6-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.2.23+rocm5.6.1-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.2.23+rocm5.6.1-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.2.23+rocm5.6.1-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/rocm/llama_cpp_python_cuda-0.2.23+rocm5.6.1-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+rocm5.6-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+rocm5.6-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+rocm5.6-cp39-cp39-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" diff --git a/requirements_amd_noavx2.txt b/requirements_amd_noavx2.txt index 89dd22e280..9d8e195a06 100644 --- a/requirements_amd_noavx2.txt +++ b/requirements_amd_noavx2.txt @@ -26,14 +26,14 @@ bitsandbytes==0.38.1; platform_system != "Windows" https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.38.1-py3-none-win_amd64.whl; platform_system == "Windows" # llama-cpp-python (CPU only, no AVX2) -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" # AMD wheels https://github.com/jllllll/AutoGPTQ/releases/download/v0.6.0/auto_gptq-0.6.0+rocm5.6-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" diff --git a/requirements_apple_intel.txt b/requirements_apple_intel.txt index d431397223..03c3859f08 100644 --- a/requirements_apple_intel.txt +++ b/requirements_apple_intel.txt @@ -26,15 +26,15 @@ bitsandbytes==0.41.1; platform_system != "Windows" https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.41.1-py3-none-win_amd64.whl; platform_system == "Windows" # Mac wheels -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.19-cp311-cp311-macosx_11_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.19-cp310-cp310-macosx_11_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.19-cp39-cp39-macosx_11_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.19-cp38-cp38-macosx_11_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp311-cp311-macosx_12_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp310-cp310-macosx_12_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp39-cp39-macosx_12_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp38-cp38-macosx_12_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp311-cp311-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp310-cp310-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp39-cp39-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp38-cp38-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp311-cp311-macosx_11_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp310-cp310-macosx_11_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp39-cp39-macosx_11_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp38-cp38-macosx_11_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp311-cp311-macosx_12_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp310-cp310-macosx_12_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp39-cp39-macosx_12_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp38-cp38-macosx_12_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp311-cp311-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp310-cp310-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp39-cp39-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp38-cp38-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.8" diff --git a/requirements_apple_silicon.txt b/requirements_apple_silicon.txt index c934353f0f..1a775a540e 100644 --- a/requirements_apple_silicon.txt +++ b/requirements_apple_silicon.txt @@ -26,19 +26,19 @@ bitsandbytes==0.41.1; platform_system != "Windows" https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.41.1-py3-none-win_amd64.whl; platform_system == "Windows" # Mac wheels -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.19-cp311-cp311-macosx_11_0_arm64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.19-cp310-cp310-macosx_11_0_arm64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.19-cp39-cp39-macosx_11_0_arm64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.19-cp38-cp38-macosx_11_0_arm64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp311-cp311-macosx_12_0_arm64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp310-cp310-macosx_12_0_arm64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp39-cp39-macosx_12_0_arm64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp38-cp38-macosx_12_0_arm64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp311-cp311-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp310-cp310-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp39-cp39-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp38-cp38-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp311-cp311-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp310-cp310-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp39-cp39-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.23-cp38-cp38-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp311-cp311-macosx_11_0_arm64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp310-cp310-macosx_11_0_arm64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp39-cp39-macosx_11_0_arm64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp38-cp38-macosx_11_0_arm64.whl; platform_system == "Darwin" and platform_release >= "20.0.0" and platform_release < "21.0.0" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp311-cp311-macosx_12_0_arm64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp310-cp310-macosx_12_0_arm64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp39-cp39-macosx_12_0_arm64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp38-cp38-macosx_12_0_arm64.whl; platform_system == "Darwin" and platform_release >= "21.0.0" and platform_release < "22.0.0" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp311-cp311-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp310-cp310-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp39-cp39-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp38-cp38-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp311-cp311-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp310-cp310-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp39-cp39-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/metal/llama_cpp_python-0.2.24-cp38-cp38-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.8" diff --git a/requirements_cpu_only.txt b/requirements_cpu_only.txt index f929e1cedd..3e5c524b31 100644 --- a/requirements_cpu_only.txt +++ b/requirements_cpu_only.txt @@ -26,11 +26,11 @@ bitsandbytes==0.41.1; platform_system != "Windows" https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.41.1-py3-none-win_amd64.whl; platform_system == "Windows" # llama-cpp-python (CPU only, AVX2) -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx2-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" diff --git a/requirements_cpu_only_noavx2.txt b/requirements_cpu_only_noavx2.txt index 50a16aa7a0..f972a7946b 100644 --- a/requirements_cpu_only_noavx2.txt +++ b/requirements_cpu_only_noavx2.txt @@ -26,11 +26,11 @@ bitsandbytes==0.41.1; platform_system != "Windows" https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.41.1-py3-none-win_amd64.whl; platform_system == "Windows" # llama-cpp-python (CPU only, no AVX2) -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" diff --git a/requirements_noavx2.txt b/requirements_noavx2.txt index e7f81b1acc..a17f8015ab 100644 --- a/requirements_noavx2.txt +++ b/requirements_noavx2.txt @@ -26,14 +26,14 @@ bitsandbytes==0.41.1; platform_system != "Windows" https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.41.1-py3-none-win_amd64.whl; platform_system == "Windows" # llama-cpp-python (CPU only, no AVX2) -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.23+cpuavx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" # CUDA wheels https://github.com/jllllll/AutoGPTQ/releases/download/v0.6.0/auto_gptq-0.6.0+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" @@ -68,14 +68,14 @@ https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp39-cp39-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp38-cp38-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121avx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121avx-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121avx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121avx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121avx-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121avx-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121avx-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" -https://github.com/jllllll/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.23+cu121avx-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" From de138b8ba67b61885fc21506f3c3ca64eda4c035 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:30:53 -0300 Subject: [PATCH 10/25] Add llama-cpp-python wheels with tensor cores support (#5003) --- README.md | 1 + modules/llamacpp_hf.py | 13 +++++++++++-- modules/llamacpp_model.py | 13 +++++++++++-- modules/loaders.py | 4 +++- modules/shared.py | 1 + modules/ui.py | 1 + modules/ui_model_menu.py | 1 + requirements.txt | 28 ++++++++++++++++++++-------- requirements_noavx2.txt | 28 ++++++++++++++++++++-------- 9 files changed, 69 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 9ff917053b..be429b1da4 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,7 @@ List of command-line flags | Flag | Description | |-------------|-------------| +| `--tensorcores` | Use llama-cpp-python compiled with tensor cores support. This increases performance on RTX cards. NVIDIA only. | | `--n_ctx N_CTX` | Size of the prompt context. | | `--threads` | Number of threads to use. | | `--threads-batch THREADS_BATCH` | Number of threads to use for batches/prompt processing. | diff --git a/modules/llamacpp_hf.py b/modules/llamacpp_hf.py index 6b702b7aeb..b8fde4d4df 100644 --- a/modules/llamacpp_hf.py +++ b/modules/llamacpp_hf.py @@ -20,12 +20,21 @@ except: llama_cpp_cuda = None +try: + import llama_cpp_cuda_tensorcores +except: + llama_cpp_cuda_tensorcores = None + def llama_cpp_lib(): - if (shared.args.cpu and llama_cpp is not None) or llama_cpp_cuda is None: + if shared.args.cpu and llama_cpp is not None: return llama_cpp - else: + elif shared.args.tensorcores and llama_cpp_cuda_tensorcores is not None: + return llama_cpp_cuda_tensorcores + elif llama_cpp_cuda is not None: return llama_cpp_cuda + else: + return llama_cpp class LlamacppHF(PreTrainedModel): diff --git a/modules/llamacpp_model.py b/modules/llamacpp_model.py index 9d29511c0d..96ea98e901 100644 --- a/modules/llamacpp_model.py +++ b/modules/llamacpp_model.py @@ -19,12 +19,21 @@ except: llama_cpp_cuda = None +try: + import llama_cpp_cuda_tensorcores +except: + llama_cpp_cuda_tensorcores = None + def llama_cpp_lib(): - if (shared.args.cpu and llama_cpp is not None) or llama_cpp_cuda is None: + if shared.args.cpu and llama_cpp is not None: return llama_cpp - else: + elif shared.args.tensorcores and llama_cpp_cuda_tensorcores is not None: + return llama_cpp_cuda_tensorcores + elif llama_cpp_cuda is not None: return llama_cpp_cuda + else: + return llama_cpp def ban_eos_logits_processor(eos_token, input_ids, logits): diff --git a/modules/loaders.py b/modules/loaders.py index ffadb037f2..cfab7dec94 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -43,7 +43,8 @@ 'compress_pos_emb', 'cpu', 'numa', - 'no_offload_kqv' + 'no_offload_kqv', + 'tensorcores', ], 'llamacpp_HF': [ 'n_ctx', @@ -65,6 +66,7 @@ 'no_use_fast', 'logits_all', 'no_offload_kqv', + 'tensorcores', 'llamacpp_HF_info', ], 'ExLlamav2_HF': [ diff --git a/modules/shared.py b/modules/shared.py index 75065d54fd..e0e77362d2 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -106,6 +106,7 @@ parser.add_argument('--quant_type', type=str, default='nf4', help='quant_type for 4-bit. Valid options: nf4, fp4.') # llama.cpp +parser.add_argument('--tensorcores', action='store_true', help='Use llama-cpp-python compiled with tensor cores support. This increases performance on RTX cards. NVIDIA only.') parser.add_argument('--n_ctx', type=int, default=2048, help='Size of the prompt context.') parser.add_argument('--threads', type=int, default=0, help='Number of threads to use.') parser.add_argument('--threads-batch', type=int, default=0, help='Number of threads to use for batches/prompt processing.') diff --git a/modules/ui.py b/modules/ui.py index 7ee42022a8..b94cceca78 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -92,6 +92,7 @@ def list_model_elements(): 'numa', 'logits_all', 'no_offload_kqv', + 'tensorcores', 'hqq_backend', ] if is_torch_xpu_available(): diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 09c743f081..4279f11842 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -105,6 +105,7 @@ def create_ui(): shared.gradio['quipsharp_info'] = gr.Markdown('QuIP# only works on Linux.') with gr.Column(): + shared.gradio['tensorcores'] = gr.Checkbox(label="tensorcores", value=shared.args.tensorcores, info='Use llama-cpp-python compiled with tensor cores support. This increases performance on RTX cards. NVIDIA only.') shared.gradio['triton'] = gr.Checkbox(label="triton", value=shared.args.triton) shared.gradio['no_inject_fused_attention'] = gr.Checkbox(label="no_inject_fused_attention", value=shared.args.no_inject_fused_attention, info='Disable fused attention. Fused attention improves inference performance but uses more VRAM. Fuses layers for AutoAWQ. Disable if running low on VRAM.') shared.gradio['no_inject_fused_mlp'] = gr.Checkbox(label="no_inject_fused_mlp", value=shared.args.no_inject_fused_mlp, info='Affects Triton only. Disable fused MLP. Fused MLP improves performance but uses more VRAM. Disable if running low on VRAM.') diff --git a/requirements.txt b/requirements.txt index 30ea3e34f2..9f2979ab6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,6 +35,26 @@ https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cp https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx2-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +# llama-cpp-python (CUDA, no tensor cores) +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" + +# llama-cpp-python (CUDA, tensor cores) +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" + # CUDA wheels https://github.com/jllllll/AutoGPTQ/releases/download/v0.6.0/auto_gptq-0.6.0+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/jllllll/AutoGPTQ/releases/download/v0.6.0/auto_gptq-0.6.0+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" @@ -68,14 +88,6 @@ https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp39-cp39-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp38-cp38-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" diff --git a/requirements_noavx2.txt b/requirements_noavx2.txt index a17f8015ab..08b85092d7 100644 --- a/requirements_noavx2.txt +++ b/requirements_noavx2.txt @@ -35,6 +35,26 @@ https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cp https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.24+cpuavx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +# llama-cpp-python (CUDA, no tensor cores) +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" + +# llama-cpp-python (CUDA, tensor cores) +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121avx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121avx-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121avx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121avx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121avx-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121avx-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121avx-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" +https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.24+cu121avx-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" + # CUDA wheels https://github.com/jllllll/AutoGPTQ/releases/download/v0.6.0/auto_gptq-0.6.0+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/jllllll/AutoGPTQ/releases/download/v0.6.0/auto_gptq-0.6.0+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" @@ -68,14 +88,6 @@ https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp39-cp39-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.4/flash_attn-2.3.4+cu122torch2.1cxx11abiFALSE-cp38-cp38-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp38-cp38-win_amd64.whl; platform_system == "Windows" and python_version == "3.8" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp311-cp311-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp310-cp310-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp39-cp39-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.9" -https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.24+cu121avx-cp38-cp38-manylinux_2_31_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.8" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10" https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp39-cp39-win_amd64.whl; platform_system == "Windows" and python_version == "3.9" From f4ae0075e8f2b8518f57888bdddedebf88793814 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 13:16:52 -0800 Subject: [PATCH 11/25] Fix conversion from old template format to jinja2 --- modules/chat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index 3106d3d24a..973a7fbdb8 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -768,13 +768,13 @@ def delete_character(name, instruct=False): def jinja_template_from_old_format(params, verbose=False): MASTER_TEMPLATE = """ -{%- set found_item = false -%} +{%- set ns = namespace(found=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} - {%- set found_item = true -%} + {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} -{%- if not found_item -%} +{%- if not ns.found -%} {{- '<|PRE-SYSTEM|>' + '<|SYSTEM-MESSAGE|>' + '<|POST-SYSTEM|>' -}} {%- endif %} {%- for message in messages %} From e83e6cedbeae98ccc8c6f91bbb79f1b0a2037155 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 13:18:26 -0800 Subject: [PATCH 12/25] Organize the model menu --- modules/ui_model_menu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 4279f11842..ae8194b1be 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -106,6 +106,7 @@ def create_ui(): with gr.Column(): shared.gradio['tensorcores'] = gr.Checkbox(label="tensorcores", value=shared.args.tensorcores, info='Use llama-cpp-python compiled with tensor cores support. This increases performance on RTX cards. NVIDIA only.') + shared.gradio['no_offload_kqv'] = gr.Checkbox(label="no_offload_kqv", value=shared.args.no_offload_kqv, info='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.') shared.gradio['triton'] = gr.Checkbox(label="triton", value=shared.args.triton) shared.gradio['no_inject_fused_attention'] = gr.Checkbox(label="no_inject_fused_attention", value=shared.args.no_inject_fused_attention, info='Disable fused attention. Fused attention improves inference performance but uses more VRAM. Fuses layers for AutoAWQ. Disable if running low on VRAM.') shared.gradio['no_inject_fused_mlp'] = gr.Checkbox(label="no_inject_fused_mlp", value=shared.args.no_inject_fused_mlp, info='Affects Triton only. Disable fused MLP. Fused MLP improves performance but uses more VRAM. Disable if running low on VRAM.') @@ -116,7 +117,6 @@ def create_ui(): shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock) shared.gradio['numa'] = gr.Checkbox(label="numa", value=shared.args.numa, info='NUMA support can help on some systems with non-uniform memory access.') shared.gradio['cpu'] = gr.Checkbox(label="cpu", value=shared.args.cpu) - shared.gradio['no_offload_kqv'] = gr.Checkbox(label="no_offload_kqv", value=shared.args.no_offload_kqv, info='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.') shared.gradio['load_in_8bit'] = gr.Checkbox(label="load-in-8bit", value=shared.args.load_in_8bit) shared.gradio['bf16'] = gr.Checkbox(label="bf16", value=shared.args.bf16) shared.gradio['auto_devices'] = gr.Checkbox(label="auto-devices", value=shared.args.auto_devices) From d8279dc71083a1b150bde340f95d17b2cb1d6a75 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:31:46 -0800 Subject: [PATCH 13/25] Replace character name placeholders in chat context (closes #5007) --- modules/chat.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/chat.py b/modules/chat.py index 973a7fbdb8..8ddb753114 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -95,7 +95,8 @@ def generate_chat_prompt(user_input, state, **kwargs): else: renderer = chat_renderer if state['context'].strip() != '': - messages.append({"role": "system", "content": state['context']}) + context = replace_character_names(state['context'], state['name1'], state['name2']) + messages.append({"role": "system", "content": context}) insert_pos = len(messages) for user_msg, assistant_msg in reversed(history): From 95600073bcd0430d2195bd82e09d82398455ae03 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 20:20:45 -0800 Subject: [PATCH 14/25] Add an informative error when extension requirements are missing --- extensions/coqui_tts/script.py | 17 ++--------------- modules/extensions.py | 7 ++++++- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/extensions/coqui_tts/script.py b/extensions/coqui_tts/script.py index 8eab7afbb2..3b241c5813 100644 --- a/extensions/coqui_tts/script.py +++ b/extensions/coqui_tts/script.py @@ -6,27 +6,14 @@ from pathlib import Path import gradio as gr +from TTS.api import TTS +from TTS.utils.synthesizer import Synthesizer from modules import chat, shared, ui_chat from modules.logging_colors import logger from modules.ui import create_refresh_button from modules.utils import gradio -try: - from TTS.api import TTS - from TTS.utils.synthesizer import Synthesizer -except ModuleNotFoundError: - logger.error( - "Could not find the TTS module. Make sure to install the requirements for the coqui_tts extension." - "\n" - "\nLinux / Mac:\npip install -r extensions/coqui_tts/requirements.txt\n" - "\nWindows:\npip install -r extensions\\coqui_tts\\requirements.txt\n" - "\n" - "If you used the one-click installer, paste the command above in the terminal window launched after running the \"cmd_\" script. On Windows, that's \"cmd_windows.bat\"." - ) - - raise - os.environ["COQUI_TOS_AGREED"] = "1" params = { diff --git a/modules/extensions.py b/modules/extensions.py index 6c0725043e..25fcc0a31d 100644 --- a/modules/extensions.py +++ b/modules/extensions.py @@ -33,7 +33,12 @@ def load_extensions(): if name != 'api': logger.info(f'Loading the extension "{name}"...') try: - exec(f"import extensions.{name}.script") + try: + exec(f"import extensions.{name}.script") + except ModuleNotFoundError: + logger.error(f"Could not import the requirements for '{name}'. Make sure to install the requirements for the extension.\n\nLinux / Mac:\n\npip install -r extensions/{name}/requirements.txt --upgrade\n\nWindows:\n\npip install -r extensions\\{name}\\requirements.txt --upgrade\n\nIf you used the one-click installer, paste the command above in the terminal window opened after launching the cmd script for your OS.") + raise + extension = getattr(extensions, name).script apply_settings(extension, name) if extension not in setup_called and hasattr(extension, "setup"): From 23818dc0981a0dbf3a49bef3beb015929f8c81f2 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 20:38:33 -0800 Subject: [PATCH 15/25] Better logger Credits: vladmandic/automatic --- extensions/openai/script.py | 2 + modules/logging_colors.py | 176 +++++++++++++----------------------- 2 files changed, 65 insertions(+), 113 deletions(-) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index 1958c30fa2..0be83442fb 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -1,5 +1,6 @@ import asyncio import json +import logging import os import traceback from threading import Thread @@ -367,6 +368,7 @@ def on_start(public_url: str): if shared.args.admin_key and shared.args.admin_key != shared.args.api_key: logger.info(f'OpenAI API admin key (for loading/unloading models):\n\n{shared.args.admin_key}\n') + logging.getLogger("uvicorn.error").propagate = False uvicorn.run(app, host=server_addr, port=port, ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile) diff --git a/modules/logging_colors.py b/modules/logging_colors.py index a0c97c3a76..ba760a2c84 100644 --- a/modules/logging_colors.py +++ b/modules/logging_colors.py @@ -1,117 +1,67 @@ -# Copied from https://stackoverflow.com/a/1336640 - import logging -import platform - -logging.basicConfig( - format='%(asctime)s %(levelname)s:%(message)s', - datefmt='%Y-%m-%d %H:%M:%S', -) - - -def add_coloring_to_emit_windows(fn): - # add methods we need to the class - def _out_handle(self): - import ctypes - return ctypes.windll.kernel32.GetStdHandle(self.STD_OUTPUT_HANDLE) - out_handle = property(_out_handle) - - def _set_color(self, code): - import ctypes - - # Constants from the Windows API - self.STD_OUTPUT_HANDLE = -11 - hdl = ctypes.windll.kernel32.GetStdHandle(self.STD_OUTPUT_HANDLE) - ctypes.windll.kernel32.SetConsoleTextAttribute(hdl, code) - - setattr(logging.StreamHandler, '_set_color', _set_color) - - def new(*args): - FOREGROUND_BLUE = 0x0001 # text color contains blue. - FOREGROUND_GREEN = 0x0002 # text color contains green. - FOREGROUND_RED = 0x0004 # text color contains red. - FOREGROUND_INTENSITY = 0x0008 # text color is intensified. - FOREGROUND_WHITE = FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED - # winbase.h - # STD_INPUT_HANDLE = -10 - # STD_OUTPUT_HANDLE = -11 - # STD_ERROR_HANDLE = -12 - - # wincon.h - # FOREGROUND_BLACK = 0x0000 - FOREGROUND_BLUE = 0x0001 - FOREGROUND_GREEN = 0x0002 - # FOREGROUND_CYAN = 0x0003 - FOREGROUND_RED = 0x0004 - FOREGROUND_MAGENTA = 0x0005 - FOREGROUND_YELLOW = 0x0006 - # FOREGROUND_GREY = 0x0007 - FOREGROUND_INTENSITY = 0x0008 # foreground color is intensified. - - # BACKGROUND_BLACK = 0x0000 - # BACKGROUND_BLUE = 0x0010 - # BACKGROUND_GREEN = 0x0020 - # BACKGROUND_CYAN = 0x0030 - # BACKGROUND_RED = 0x0040 - # BACKGROUND_MAGENTA = 0x0050 - BACKGROUND_YELLOW = 0x0060 - # BACKGROUND_GREY = 0x0070 - BACKGROUND_INTENSITY = 0x0080 # background color is intensified. - - levelno = args[1].levelno - if (levelno >= 50): - color = BACKGROUND_YELLOW | FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_INTENSITY - elif (levelno >= 40): - color = FOREGROUND_RED | FOREGROUND_INTENSITY - elif (levelno >= 30): - color = FOREGROUND_YELLOW | FOREGROUND_INTENSITY - elif (levelno >= 20): - color = FOREGROUND_GREEN - elif (levelno >= 10): - color = FOREGROUND_MAGENTA - else: - color = FOREGROUND_WHITE - args[0]._set_color(color) - - ret = fn(*args) - args[0]._set_color(FOREGROUND_WHITE) - # print "after" - return ret - return new - - -def add_coloring_to_emit_ansi(fn): - # add methods we need to the class - def new(*args): - levelno = args[1].levelno - if (levelno >= 50): - color = '\x1b[31m' # red - elif (levelno >= 40): - color = '\x1b[31m' # red - elif (levelno >= 30): - color = '\x1b[33m' # yellow - elif (levelno >= 20): - color = '\x1b[32m' # green - elif (levelno >= 10): - color = '\x1b[35m' # pink - else: - color = '\x1b[0m' # normal - args[1].msg = color + args[1].msg + '\x1b[0m' # normal - # print "after" - return fn(*args) - return new +logger = logging.getLogger('text-generation-webui') -if platform.system() == 'Windows': - # Windows does not support ANSI escapes and we are using API calls to set the console color - logging.StreamHandler.emit = add_coloring_to_emit_windows(logging.StreamHandler.emit) -else: - # all non-Windows platforms are supporting ANSI escapes so we use them - logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) - # log = logging.getLogger() - # log.addFilter(log_filter()) - # //hdlr = logging.StreamHandler() - # //hdlr.setFormatter(formatter()) -logger = logging.getLogger('text-generation-webui') -logger.setLevel(logging.DEBUG) +def setup_logging(): + ''' + Copied from: https://github.com/vladmandic/automatic + + All credits to vladmandic. + ''' + + class RingBuffer(logging.StreamHandler): + def __init__(self, capacity): + super().__init__() + self.capacity = capacity + self.buffer = [] + self.formatter = logging.Formatter('{ "asctime":"%(asctime)s", "created":%(created)f, "facility":"%(name)s", "pid":%(process)d, "tid":%(thread)d, "level":"%(levelname)s", "module":"%(module)s", "func":"%(funcName)s", "msg":"%(message)s" }') + + def emit(self, record): + msg = self.format(record) + # self.buffer.append(json.loads(msg)) + self.buffer.append(msg) + if len(self.buffer) > self.capacity: + self.buffer.pop(0) + + def get(self): + return self.buffer + + from rich.theme import Theme + from rich.logging import RichHandler + from rich.console import Console + from rich.pretty import install as pretty_install + from rich.traceback import install as traceback_install + + level = logging.DEBUG + logger.setLevel(logging.DEBUG) # log to file is always at level debug for facility `sd` + console = Console(log_time=True, log_time_format='%H:%M:%S-%f', theme=Theme({ + "traceback.border": "black", + "traceback.border.syntax_error": "black", + "inspect.value.border": "black", + })) + logging.basicConfig(level=logging.ERROR, format='%(asctime)s | %(name)s | %(levelname)s | %(module)s | %(message)s', handlers=[logging.NullHandler()]) # redirect default logger to null + pretty_install(console=console) + traceback_install(console=console, extra_lines=1, max_frames=10, width=console.width, word_wrap=False, indent_guides=False, suppress=[]) + while logger.hasHandlers() and len(logger.handlers) > 0: + logger.removeHandler(logger.handlers[0]) + + # handlers + rh = RichHandler(show_time=True, omit_repeated_times=False, show_level=True, show_path=False, markup=False, rich_tracebacks=True, log_time_format='%H:%M:%S-%f', level=level, console=console) + rh.setLevel(level) + logger.addHandler(rh) + + rb = RingBuffer(100) # 100 entries default in log ring buffer + rb.setLevel(level) + logger.addHandler(rb) + logger.buffer = rb.buffer + + # overrides + logging.getLogger("urllib3").setLevel(logging.ERROR) + logging.getLogger("httpx").setLevel(logging.ERROR) + logging.getLogger("diffusers").setLevel(logging.ERROR) + logging.getLogger("torch").setLevel(logging.ERROR) + logging.getLogger("lycoris").handlers = logger.handlers + + +setup_logging() From 9992f7d8c0b35c6e076e4e3cdd726cbd63d56b85 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 20:54:32 -0800 Subject: [PATCH 16/25] Improve several log messages --- modules/GPTQ_loader.py | 2 +- modules/LoRA.py | 2 +- modules/extensions.py | 2 +- modules/models.py | 2 +- modules/shared.py | 34 +++++++++++++++++++--------------- modules/training.py | 18 +++++++++--------- server.py | 7 ++++++- 7 files changed, 38 insertions(+), 29 deletions(-) diff --git a/modules/GPTQ_loader.py b/modules/GPTQ_loader.py index 7dc20b0a0e..601c58f30a 100644 --- a/modules/GPTQ_loader.py +++ b/modules/GPTQ_loader.py @@ -126,7 +126,7 @@ def load_quantized(model_name): path_to_model = Path(f'{shared.args.model_dir}/{model_name}') pt_path = find_quantized_model_file(model_name) if not pt_path: - logger.error("Could not find the quantized model in .pt or .safetensors format, exiting...") + logger.error("Could not find the quantized model in .pt or .safetensors format. Exiting.") exit() else: logger.info(f"Found the following quantized model: {pt_path}") diff --git a/modules/LoRA.py b/modules/LoRA.py index dea476ad4e..97027eb4c8 100644 --- a/modules/LoRA.py +++ b/modules/LoRA.py @@ -138,7 +138,7 @@ def add_lora_transformers(lora_names): # Add a LoRA when another LoRA is already present if len(removed_set) == 0 and len(prior_set) > 0 and "__merged" not in shared.model.peft_config.keys(): - logger.info(f"Adding the LoRA(s) named {added_set} to the model...") + logger.info(f"Adding the LoRA(s) named {added_set} to the model") for lora in added_set: shared.model.load_adapter(get_lora_path(lora), lora) diff --git a/modules/extensions.py b/modules/extensions.py index 25fcc0a31d..2a3b0bb1d4 100644 --- a/modules/extensions.py +++ b/modules/extensions.py @@ -31,7 +31,7 @@ def load_extensions(): for i, name in enumerate(shared.args.extensions): if name in available_extensions: if name != 'api': - logger.info(f'Loading the extension "{name}"...') + logger.info(f'Loading the extension "{name}"') try: try: exec(f"import extensions.{name}.script") diff --git a/modules/models.py b/modules/models.py index 5a23f7433e..f37f3d60a6 100644 --- a/modules/models.py +++ b/modules/models.py @@ -54,7 +54,7 @@ def load_model(model_name, loader=None): - logger.info(f"Loading {model_name}...") + logger.info(f"Loading {model_name}") t0 = time.time() shared.is_seq2seq = False diff --git a/modules/shared.py b/modules/shared.py index e0e77362d2..bcb2090543 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -204,22 +204,26 @@ if hasattr(args, arg): provided_arguments.append(arg) -# Deprecation warnings deprecated_args = ['notebook', 'chat', 'no_stream', 'mul_mat_q', 'use_fast'] -for k in deprecated_args: - if getattr(args, k): - logger.warning(f'The --{k} flag has been deprecated and will be removed soon. Please remove that flag.') - -# Security warnings -if args.trust_remote_code: - logger.warning('trust_remote_code is enabled. This is dangerous.') -if 'COLAB_GPU' not in os.environ and not args.nowebui: - if args.share: - logger.warning("The gradio \"share link\" feature uses a proprietary executable to create a reverse tunnel. Use it with care.") - if any((args.listen, args.share)) and not any((args.gradio_auth, args.gradio_auth_path)): - logger.warning("\nYou are potentially exposing the web UI to the entire internet without any access password.\nYou can create one with the \"--gradio-auth\" flag like this:\n\n--gradio-auth username:password\n\nMake sure to replace username:password with your own.") - if args.multi_user: - logger.warning('\nThe multi-user mode is highly experimental and should not be shared publicly.') + + +def do_cmd_flags_warnings(): + + # Deprecation warnings + for k in deprecated_args: + if getattr(args, k): + logger.warning(f'The --{k} flag has been deprecated and will be removed soon. Please remove that flag.') + + # Security warnings + if args.trust_remote_code: + logger.warning('trust_remote_code is enabled. This is dangerous.') + if 'COLAB_GPU' not in os.environ and not args.nowebui: + if args.share: + logger.warning("The gradio \"share link\" feature uses a proprietary executable to create a reverse tunnel. Use it with care.") + if any((args.listen, args.share)) and not any((args.gradio_auth, args.gradio_auth_path)): + logger.warning("\nYou are potentially exposing the web UI to the entire internet without any access password.\nYou can create one with the \"--gradio-auth\" flag like this:\n\n--gradio-auth username:password\n\nMake sure to replace username:password with your own.") + if args.multi_user: + logger.warning('\nThe multi-user mode is highly experimental and should not be shared publicly.') def fix_loader_name(name): diff --git a/modules/training.py b/modules/training.py index ca1fffb366..b0e0240085 100644 --- a/modules/training.py +++ b/modules/training.py @@ -249,7 +249,7 @@ def backup_adapter(input_folder): adapter_file = Path(f"{input_folder}/adapter_model.bin") if adapter_file.is_file(): - logger.info("Backing up existing LoRA adapter...") + logger.info("Backing up existing LoRA adapter") creation_date = datetime.fromtimestamp(adapter_file.stat().st_ctime) creation_date_str = creation_date.strftime("Backup-%Y-%m-%d") @@ -406,7 +406,7 @@ def tokenize(prompt, append_eos_token=False): # == Prep the dataset, format, etc == if raw_text_file not in ['None', '']: train_template["template_type"] = "raw_text" - logger.info("Loading raw text file dataset...") + logger.info("Loading raw text file dataset") fullpath = clean_path('training/datasets', f'{raw_text_file}') fullpath = Path(fullpath) if fullpath.is_dir(): @@ -486,7 +486,7 @@ def generate_and_tokenize_prompt(data_point): prompt = generate_prompt(data_point) return tokenize(prompt, add_eos_token) - logger.info("Loading JSON datasets...") + logger.info("Loading JSON datasets") data = load_dataset("json", data_files=clean_path('training/datasets', f'{dataset}.json')) train_data = data['train'].map(generate_and_tokenize_prompt, new_fingerprint='%030x' % random.randrange(16**30)) @@ -516,13 +516,13 @@ def generate_and_tokenize_prompt(data_point): # == Start prepping the model itself == if not hasattr(shared.model, 'lm_head') or hasattr(shared.model.lm_head, 'weight'): - logger.info("Getting model ready...") + logger.info("Getting model ready") prepare_model_for_kbit_training(shared.model) # base model is now frozen and should not be reused for any other LoRA training than this one shared.model_dirty_from_training = True - logger.info("Preparing for training...") + logger.info("Preparing for training") config = LoraConfig( r=lora_rank, lora_alpha=lora_alpha, @@ -540,10 +540,10 @@ def generate_and_tokenize_prompt(data_point): model_trainable_params, model_all_params = calc_trainable_parameters(shared.model) try: - logger.info("Creating LoRA model...") + logger.info("Creating LoRA model") lora_model = get_peft_model(shared.model, config) if not always_override and Path(f"{lora_file_path}/adapter_model.bin").is_file(): - logger.info("Loading existing LoRA data...") + logger.info("Loading existing LoRA data") state_dict_peft = torch.load(f"{lora_file_path}/adapter_model.bin", weights_only=True) set_peft_model_state_dict(lora_model, state_dict_peft) except: @@ -648,7 +648,7 @@ def on_log(self, args: transformers.TrainingArguments, state: transformers.Train json.dump(train_template, file, indent=2) # == Main run and monitor loop == - logger.info("Starting training...") + logger.info("Starting training") yield "Starting..." lora_trainable_param, lora_all_param = calc_trainable_parameters(lora_model) @@ -730,7 +730,7 @@ def threaded_run(): # Saving in the train thread might fail if an error occurs, so save here if so. if not tracked.did_save: - logger.info("Training complete, saving...") + logger.info("Training complete, saving") lora_model.save_pretrained(lora_file_path) if WANT_INTERRUPT: diff --git a/server.py b/server.py index ae0aed0955..d5d11bc42a 100644 --- a/server.py +++ b/server.py @@ -12,6 +12,7 @@ warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated') warnings.filterwarnings('ignore', category=UserWarning, message='Using the update method is deprecated') warnings.filterwarnings('ignore', category=UserWarning, message='Field "model_name" has conflict') +warnings.filterwarnings('ignore', category=UserWarning, message='The value passed into gr.Dropdown()') with RequestBlocker(): import gradio as gr @@ -54,6 +55,7 @@ get_model_metadata, update_model_parameters ) +from modules.shared import do_cmd_flags_warnings from modules.utils import gradio @@ -170,6 +172,9 @@ def create_interface(): if __name__ == "__main__": + logger.info("Starting Text generation web UI") + do_cmd_flags_warnings() + # Load custom settings settings_file = None if shared.args.settings is not None and Path(shared.args.settings).exists(): @@ -180,7 +185,7 @@ def create_interface(): settings_file = Path('settings.json') if settings_file is not None: - logger.info(f"Loading settings from {settings_file}...") + logger.info(f"Loading settings from {settings_file}") file_contents = open(settings_file, 'r', encoding='utf-8').read() new_settings = json.loads(file_contents) if settings_file.suffix == "json" else yaml.safe_load(file_contents) shared.settings.update(new_settings) From 366c93a008cb2c0cf23e88d6bdbb757626537688 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:03:20 -0800 Subject: [PATCH 17/25] Hide a warning --- server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server.py b/server.py index d5d11bc42a..c53882f6d8 100644 --- a/server.py +++ b/server.py @@ -13,6 +13,7 @@ warnings.filterwarnings('ignore', category=UserWarning, message='Using the update method is deprecated') warnings.filterwarnings('ignore', category=UserWarning, message='Field "model_name" has conflict') warnings.filterwarnings('ignore', category=UserWarning, message='The value passed into gr.Dropdown()') +warnings.filterwarnings('ignore', category=UserWarning, message='Field "model_names" has conflict') with RequestBlocker(): import gradio as gr From fb8ee9f7ff164aff95a747c73d3924e9613a76b8 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:32:58 -0800 Subject: [PATCH 18/25] Add a specific error if HQQ is missing --- modules/models.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/models.py b/modules/models.py index f37f3d60a6..7031695217 100644 --- a/modules/models.py +++ b/modules/models.py @@ -413,8 +413,12 @@ def ExLlamav2_HF_loader(model_name): def HQQ_loader(model_name): - from hqq.engine.hf import HQQModelForCausalLM - from hqq.core.quantize import HQQLinear, HQQBackend + try: + from hqq.engine.hf import HQQModelForCausalLM + from hqq.core.quantize import HQQLinear, HQQBackend + except ModuleNotFoundError: + logger.error("HQQ is not installed. You can install it with:\n\npip install hqq") + return None logger.info(f"Loading HQQ model with backend: {shared.args.hqq_backend}") From 2289e9031e50326ddfae962db6e7f3cc6225077f Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:33:49 -0800 Subject: [PATCH 19/25] Remove HQQ from requirements (after https://github.com/oobabooga/text-generation-webui/issues/4993) --- requirements.txt | 1 - requirements_amd.txt | 1 - requirements_amd_noavx2.txt | 1 - requirements_apple_intel.txt | 1 - requirements_apple_silicon.txt | 1 - requirements_cpu_only.txt | 1 - requirements_cpu_only_noavx2.txt | 1 - requirements_noavx2.txt | 1 - requirements_nowheels.txt | 1 - 9 files changed, 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9f2979ab6b..c7c6edaa23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ datasets einops exllamav2==0.0.11; platform_system != "Darwin" and platform_machine != "x86_64" gradio==3.50.* -hqq==0.1.1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_amd.txt b/requirements_amd.txt index dcdf69869f..6972b4aa3f 100644 --- a/requirements_amd.txt +++ b/requirements_amd.txt @@ -4,7 +4,6 @@ datasets einops exllamav2==0.0.11; platform_system == "Windows" or python_version < "3.10" or python_version > "3.11" or platform_machine != "x86_64" gradio==3.50.* -hqq==0.1.1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_amd_noavx2.txt b/requirements_amd_noavx2.txt index 9d8e195a06..af58c5c513 100644 --- a/requirements_amd_noavx2.txt +++ b/requirements_amd_noavx2.txt @@ -4,7 +4,6 @@ datasets einops exllamav2==0.0.11; platform_system == "Windows" or python_version < "3.10" or python_version > "3.11" or platform_machine != "x86_64" gradio==3.50.* -hqq==0.1.1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_apple_intel.txt b/requirements_apple_intel.txt index 03c3859f08..a4147217b2 100644 --- a/requirements_apple_intel.txt +++ b/requirements_apple_intel.txt @@ -4,7 +4,6 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* -hqq==0.1.1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_apple_silicon.txt b/requirements_apple_silicon.txt index 1a775a540e..d36c7d1b7f 100644 --- a/requirements_apple_silicon.txt +++ b/requirements_apple_silicon.txt @@ -4,7 +4,6 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* -hqq==0.1.1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_cpu_only.txt b/requirements_cpu_only.txt index 3e5c524b31..c6b1a25499 100644 --- a/requirements_cpu_only.txt +++ b/requirements_cpu_only.txt @@ -4,7 +4,6 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* -hqq==0.1.1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_cpu_only_noavx2.txt b/requirements_cpu_only_noavx2.txt index f972a7946b..c442e525f6 100644 --- a/requirements_cpu_only_noavx2.txt +++ b/requirements_cpu_only_noavx2.txt @@ -4,7 +4,6 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* -hqq==0.1.1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_noavx2.txt b/requirements_noavx2.txt index 08b85092d7..0d92f4140c 100644 --- a/requirements_noavx2.txt +++ b/requirements_noavx2.txt @@ -4,7 +4,6 @@ datasets einops exllamav2==0.0.11; platform_system != "Darwin" and platform_machine != "x86_64" gradio==3.50.* -hqq==0.1.1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_nowheels.txt b/requirements_nowheels.txt index cabccf7c13..bc5cadcb2b 100644 --- a/requirements_nowheels.txt +++ b/requirements_nowheels.txt @@ -4,7 +4,6 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* -hqq==0.1.1 markdown numpy==1.24.* optimum==1.16.* From fadb295d4dbec37d806d3b8fd922cddb976adbdc Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:36:57 -0800 Subject: [PATCH 20/25] Lint --- extensions/coqui_tts/script.py | 1 - modules/logging_colors.py | 4 ++-- modules/models.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/extensions/coqui_tts/script.py b/extensions/coqui_tts/script.py index 3b241c5813..26d6b76a61 100644 --- a/extensions/coqui_tts/script.py +++ b/extensions/coqui_tts/script.py @@ -10,7 +10,6 @@ from TTS.utils.synthesizer import Synthesizer from modules import chat, shared, ui_chat -from modules.logging_colors import logger from modules.ui import create_refresh_button from modules.utils import gradio diff --git a/modules/logging_colors.py b/modules/logging_colors.py index ba760a2c84..b9791e2685 100644 --- a/modules/logging_colors.py +++ b/modules/logging_colors.py @@ -27,10 +27,10 @@ def emit(self, record): def get(self): return self.buffer - from rich.theme import Theme - from rich.logging import RichHandler from rich.console import Console + from rich.logging import RichHandler from rich.pretty import install as pretty_install + from rich.theme import Theme from rich.traceback import install as traceback_install level = logging.DEBUG diff --git a/modules/models.py b/modules/models.py index 7031695217..cad6a165b8 100644 --- a/modules/models.py +++ b/modules/models.py @@ -414,8 +414,8 @@ def ExLlamav2_HF_loader(model_name): def HQQ_loader(model_name): try: + from hqq.core.quantize import HQQBackend, HQQLinear from hqq.engine.hf import HQQModelForCausalLM - from hqq.core.quantize import HQQLinear, HQQBackend except ModuleNotFoundError: logger.error("HQQ is not installed. You can install it with:\n\npip install hqq") return None From 258c695ead3ae8dfcfdd7bdd68f3257ebbf04148 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:58:36 -0800 Subject: [PATCH 21/25] Add rich requirement --- requirements.txt | 1 + requirements_amd.txt | 1 + requirements_amd_noavx2.txt | 1 + requirements_apple_intel.txt | 1 + requirements_apple_silicon.txt | 1 + requirements_cpu_only.txt | 1 + requirements_cpu_only_noavx2.txt | 1 + requirements_noavx2.txt | 1 + requirements_nowheels.txt | 1 + 9 files changed, 9 insertions(+) diff --git a/requirements.txt b/requirements.txt index c7c6edaa23..38f8efdb56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ peft==0.7.* Pillow>=9.5.0 pyyaml requests +rich safetensors==0.4.1 scipy sentencepiece diff --git a/requirements_amd.txt b/requirements_amd.txt index 6972b4aa3f..458d810d8f 100644 --- a/requirements_amd.txt +++ b/requirements_amd.txt @@ -12,6 +12,7 @@ peft==0.7.* Pillow>=9.5.0 pyyaml requests +rich safetensors==0.4.1 scipy sentencepiece diff --git a/requirements_amd_noavx2.txt b/requirements_amd_noavx2.txt index af58c5c513..a4bb1551aa 100644 --- a/requirements_amd_noavx2.txt +++ b/requirements_amd_noavx2.txt @@ -12,6 +12,7 @@ peft==0.7.* Pillow>=9.5.0 pyyaml requests +rich safetensors==0.4.1 scipy sentencepiece diff --git a/requirements_apple_intel.txt b/requirements_apple_intel.txt index a4147217b2..56eccd35ac 100644 --- a/requirements_apple_intel.txt +++ b/requirements_apple_intel.txt @@ -12,6 +12,7 @@ peft==0.7.* Pillow>=9.5.0 pyyaml requests +rich safetensors==0.4.1 scipy sentencepiece diff --git a/requirements_apple_silicon.txt b/requirements_apple_silicon.txt index d36c7d1b7f..54606b0179 100644 --- a/requirements_apple_silicon.txt +++ b/requirements_apple_silicon.txt @@ -12,6 +12,7 @@ peft==0.7.* Pillow>=9.5.0 pyyaml requests +rich safetensors==0.4.1 scipy sentencepiece diff --git a/requirements_cpu_only.txt b/requirements_cpu_only.txt index c6b1a25499..09936b7465 100644 --- a/requirements_cpu_only.txt +++ b/requirements_cpu_only.txt @@ -12,6 +12,7 @@ peft==0.7.* Pillow>=9.5.0 pyyaml requests +rich safetensors==0.4.1 scipy sentencepiece diff --git a/requirements_cpu_only_noavx2.txt b/requirements_cpu_only_noavx2.txt index c442e525f6..65734de47a 100644 --- a/requirements_cpu_only_noavx2.txt +++ b/requirements_cpu_only_noavx2.txt @@ -12,6 +12,7 @@ peft==0.7.* Pillow>=9.5.0 pyyaml requests +rich safetensors==0.4.1 scipy sentencepiece diff --git a/requirements_noavx2.txt b/requirements_noavx2.txt index 0d92f4140c..6c8579f399 100644 --- a/requirements_noavx2.txt +++ b/requirements_noavx2.txt @@ -12,6 +12,7 @@ peft==0.7.* Pillow>=9.5.0 pyyaml requests +rich safetensors==0.4.1 scipy sentencepiece diff --git a/requirements_nowheels.txt b/requirements_nowheels.txt index bc5cadcb2b..d9a689f9d8 100644 --- a/requirements_nowheels.txt +++ b/requirements_nowheels.txt @@ -12,6 +12,7 @@ peft==0.7.* Pillow>=9.5.0 pyyaml requests +rich safetensors==0.4.1 scipy sentencepiece From b15f51015477c9709e2dff616c20466e9b3dc727 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 20 Dec 2023 07:31:42 -0800 Subject: [PATCH 22/25] Optimize ExLlamav2 (non-HF) loader --- modules/exllamav2.py | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/modules/exllamav2.py b/modules/exllamav2.py index 2cf4a03971..3a6b231aab 100644 --- a/modules/exllamav2.py +++ b/modules/exllamav2.py @@ -1,4 +1,3 @@ -import random import traceback from pathlib import Path @@ -10,7 +9,7 @@ ExLlamaV2Config, ExLlamaV2Tokenizer ) -from exllamav2.generator import ExLlamaV2BaseGenerator, ExLlamaV2Sampler +from exllamav2.generator import ExLlamaV2Sampler, ExLlamaV2StreamingGenerator from modules import shared from modules.logging_colors import logger @@ -64,7 +63,7 @@ def from_pretrained(self, path_to_model): else: cache = ExLlamaV2Cache(model) - generator = ExLlamaV2BaseGenerator(model, cache, tokenizer) + generator = ExLlamaV2StreamingGenerator(model, cache, tokenizer) result = self() result.model = model @@ -115,41 +114,22 @@ def generate_with_streaming(self, prompt, state): ids = self.tokenizer.encode(prompt, add_bos=state['add_bos_token'], encode_special_tokens=True) ids = ids[:, -get_max_prompt_length(state):] - initial_len = ids.shape[-1] if state['auto_max_new_tokens']: max_new_tokens = state['truncation_length'] - ids.shape[-1] else: max_new_tokens = state['max_new_tokens'] - # _gen_begin_base - self.cache.current_seq_len = 0 - self.model.forward(ids[:, :-1], self.cache, input_mask=None, preprocess_only=True, loras=self.loras) + self.generator.set_stop_conditions([]) + self.generator.begin_stream(ids, settings, loras=self.loras) - has_leading_space = False + decoded_text = '' for i in range(max_new_tokens): - logits = self.model.forward(ids[:, -1:], self.cache, input_mask=None, loras=self.loras).float().cpu() - token, _, _ = ExLlamaV2Sampler.sample(logits, settings, ids, random.random(), self.tokenizer) - ids = torch.cat([ids, token], dim=1) - - if i == 0 and self.tokenizer.tokenizer.id_to_piece(int(token)).startswith('▁'): - has_leading_space = True - - decoded_text = self.tokenizer.decode(ids[:, initial_len:], decode_special_tokens=not state['skip_special_tokens'])[0] - if has_leading_space: - decoded_text = ' ' + decoded_text - - # Check the partial unicode character - if chr(0xfffd) in decoded_text: - is_last = i == max_new_tokens - 1 - is_stopping = token.item() == self.tokenizer.eos_token_id or shared.stop_everything - # If we are not at the end of the generation, we skip this token - if not (is_last or is_stopping): - continue - - if token.item() == self.tokenizer.eos_token_id or shared.stop_everything: + chunk, eos, _ = self.generator.stream() + if eos or shared.stop_everything: break + decoded_text += chunk yield decoded_text def generate(self, prompt, state): From f0f6d9bdf9f6fbd41965d619c9359de65c0a7d10 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 20 Dec 2023 07:36:33 -0800 Subject: [PATCH 23/25] Add HQQ back & update version This reverts commit 2289e9031e50326ddfae962db6e7f3cc6225077f. --- modules/models.py | 8 ++------ requirements.txt | 1 + requirements_amd.txt | 1 + requirements_amd_noavx2.txt | 1 + requirements_apple_intel.txt | 1 + requirements_apple_silicon.txt | 1 + requirements_cpu_only.txt | 1 + requirements_cpu_only_noavx2.txt | 1 + requirements_noavx2.txt | 1 + requirements_nowheels.txt | 1 + 10 files changed, 11 insertions(+), 6 deletions(-) diff --git a/modules/models.py b/modules/models.py index cad6a165b8..7a1124d1cd 100644 --- a/modules/models.py +++ b/modules/models.py @@ -413,12 +413,8 @@ def ExLlamav2_HF_loader(model_name): def HQQ_loader(model_name): - try: - from hqq.core.quantize import HQQBackend, HQQLinear - from hqq.engine.hf import HQQModelForCausalLM - except ModuleNotFoundError: - logger.error("HQQ is not installed. You can install it with:\n\npip install hqq") - return None + from hqq.core.quantize import HQQBackend, HQQLinear + from hqq.engine.hf import HQQModelForCausalLM logger.info(f"Loading HQQ model with backend: {shared.args.hqq_backend}") diff --git a/requirements.txt b/requirements.txt index 38f8efdb56..4843741bba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11; platform_system != "Darwin" and platform_machine != "x86_64" gradio==3.50.* +hqq==0.1.1.post1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_amd.txt b/requirements_amd.txt index 458d810d8f..f15014adf5 100644 --- a/requirements_amd.txt +++ b/requirements_amd.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11; platform_system == "Windows" or python_version < "3.10" or python_version > "3.11" or platform_machine != "x86_64" gradio==3.50.* +hqq==0.1.1.post1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_amd_noavx2.txt b/requirements_amd_noavx2.txt index a4bb1551aa..843cbac1e7 100644 --- a/requirements_amd_noavx2.txt +++ b/requirements_amd_noavx2.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11; platform_system == "Windows" or python_version < "3.10" or python_version > "3.11" or platform_machine != "x86_64" gradio==3.50.* +hqq==0.1.1.post1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_apple_intel.txt b/requirements_apple_intel.txt index 56eccd35ac..cee6d185b7 100644 --- a/requirements_apple_intel.txt +++ b/requirements_apple_intel.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1.post1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_apple_silicon.txt b/requirements_apple_silicon.txt index 54606b0179..a3aede26a2 100644 --- a/requirements_apple_silicon.txt +++ b/requirements_apple_silicon.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1.post1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_cpu_only.txt b/requirements_cpu_only.txt index 09936b7465..af04acf732 100644 --- a/requirements_cpu_only.txt +++ b/requirements_cpu_only.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1.post1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_cpu_only_noavx2.txt b/requirements_cpu_only_noavx2.txt index 65734de47a..1c9d15c07f 100644 --- a/requirements_cpu_only_noavx2.txt +++ b/requirements_cpu_only_noavx2.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1.post1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_noavx2.txt b/requirements_noavx2.txt index 6c8579f399..39751fc5e5 100644 --- a/requirements_noavx2.txt +++ b/requirements_noavx2.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11; platform_system != "Darwin" and platform_machine != "x86_64" gradio==3.50.* +hqq==0.1.1.post1 markdown numpy==1.24.* optimum==1.16.* diff --git a/requirements_nowheels.txt b/requirements_nowheels.txt index d9a689f9d8..22e10c6b07 100644 --- a/requirements_nowheels.txt +++ b/requirements_nowheels.txt @@ -4,6 +4,7 @@ datasets einops exllamav2==0.0.11 gradio==3.50.* +hqq==0.1.1.post1 markdown numpy==1.24.* optimum==1.16.* From bcba200790bcf44164c83db7a8eb2c81e06285c1 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 20 Dec 2023 07:54:06 -0800 Subject: [PATCH 24/25] Fix EOS being ignored in ExLlamav2 after previous commit --- modules/exllamav2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/exllamav2.py b/modules/exllamav2.py index 3a6b231aab..2730d9f521 100644 --- a/modules/exllamav2.py +++ b/modules/exllamav2.py @@ -120,7 +120,6 @@ def generate_with_streaming(self, prompt, state): else: max_new_tokens = state['max_new_tokens'] - self.generator.set_stop_conditions([]) self.generator.begin_stream(ids, settings, loras=self.loras) decoded_text = '' From 6efbe3009fb7accb10de6f4777c8aa7edc5cb65e Mon Sep 17 00:00:00 2001 From: luna Date: Wed, 20 Dec 2023 13:29:19 -0300 Subject: [PATCH 25/25] let exllama v1 models load safetensor loras (#4854) --- modules/LoRA.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/LoRA.py b/modules/LoRA.py index 97027eb4c8..be2a7c75f8 100644 --- a/modules/LoRA.py +++ b/modules/LoRA.py @@ -53,7 +53,10 @@ def add_lora_exllama(lora_names): lora_path = get_lora_path(lora_names[0]) lora_config_path = lora_path / "adapter_config.json" - lora_adapter_path = lora_path / "adapter_model.bin" + for file_name in ["adapter_model.safetensors", "adapter_model.bin"]: + file_path = lora_path / file_name + if file_path.is_file(): + lora_adapter_path = file_path logger.info("Applying the following LoRAs to {}: {}".format(shared.model_name, ', '.join([lora_names[0]]))) if shared.model.__class__.__name__ == 'ExllamaModel':