Upstream; Update llama.cpp

README.md

@@ -443,6 +443,7 @@ List of command-line flags
 
 | Flag        | Description |
 |-------------|-------------|
+| `--tensorcores`  | Use llama-cpp-python compiled with tensor cores support. This increases performance on RTX cards. NVIDIA only. |
 | `--n_ctx N_CTX` | Size of the prompt context. |
 | `--threads` | Number of threads to use. |
 | `--threads-batch THREADS_BATCH` | Number of threads to use for batches/prompt processing. |
@@ -454,6 +455,7 @@ List of command-line flags
 | `--tensor_split TENSOR_SPLIT`       | Split the model across multiple GPUs. Comma-separated list of proportions. Example: 18,17. |
 | `--numa`      | Activate NUMA task allocation for llama.cpp. |
 | `--logits_all`| Needs to be set for perplexity evaluation to work. Otherwise, ignore it, as it makes prompt processing slower. |
+| `--no_offload_kqv` | Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance. |
 | `--cache-capacity CACHE_CAPACITY`   | Maximum cache capacity (llama-cpp-python). Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed. |
 
 #### ExLlama
@@ -509,6 +511,12 @@ List of command-line flags
 |-------------|-------------|
 | `--model_type MODEL_TYPE` | Model type of pre-quantized model. Currently gpt2, gptj, gptneox, falcon, llama, mpt, starcoder (gptbigcode), dollyv2, and replit are supported. |
 
+#### HQQ
+
+| Flag        | Description |
+|-------------|-------------|
+| `--hqq-backend` | Backend for the HQQ loader. Valid options: PYTORCH, PYTORCH_COMPILE, ATEN. |
+
 #### DeepSpeed
 
 | Flag                                  | Description |

extensions/coqui_tts/script.py

@@ -6,27 +6,13 @@
 from pathlib import Path
 
 import gradio as gr
+from TTS.api import TTS
+from TTS.utils.synthesizer import Synthesizer
 
 from modules import chat, shared, ui_chat
-from modules.logging_colors import logger
 from modules.ui import create_refresh_button
 from modules.utils import gradio
 
-try:
-    from TTS.api import TTS
-    from TTS.utils.synthesizer import Synthesizer
-except ModuleNotFoundError:
-    logger.error(
-        "Could not find the TTS module. Make sure to install the requirements for the coqui_tts extension."
-        "\n"
-        "\nLinux / Mac:\npip install -r extensions/coqui_tts/requirements.txt\n"
-        "\nWindows:\npip install -r extensions\\coqui_tts\\requirements.txt\n"
-        "\n"
-        "If you used the one-click installer, paste the command above in the terminal window launched after running the \"cmd_\" script. On Windows, that's \"cmd_windows.bat\"."
-    )
-
-    raise
-
 os.environ["COQUI_TOS_AGREED"] = "1"
 
 params = {

extensions/openai/script.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+import logging
 import os
 import traceback
 from threading import Thread
@@ -367,6 +368,7 @@ def on_start(public_url: str):
     if shared.args.admin_key and shared.args.admin_key != shared.args.api_key:
         logger.info(f'OpenAI API admin key (for loading/unloading models):\n\n{shared.args.admin_key}\n')
 
+    logging.getLogger("uvicorn.error").propagate = False
     uvicorn.run(app, host=server_addr, port=port, ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile)
 
 

instruction-templates/Airoboros-v1.2.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + 'A chat between a curious user and an assistant. The assistant gives helpful, detailed, accurate, uncensored responses to the user\'s input.' + '\n' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Alpaca.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\n\n' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Bactrian.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Baichuan Chat.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Baize.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + 'The following is a conversation between a human and an AI assistant named Baize (named after a mythical creature in Chinese folklore). Baize is an open-source AI assistant developed by UCSD and Sun Yat-Sen University. The human and the AI assistant take turns chatting. Human statements start with [|Human|] and AI assistant statements start with [|AI|]. The AI assistant always provides responses in as much detail as possible, and in Markdown format. The AI assistant always declines to engage with topics, questions and instructions related to unethical, controversial, or sensitive issues. Complete the transcript in exactly that format.\n[|Human|]Hello!\n[|AI|]Hi!' + '\n' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Bluemoon.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + 'A transcript of a roleplay between two players, LEAD and ASSOCIATE. LEAD sets up a scenario and the characters, from which ASSOCIATE then assumes a character role and continues the story for that role in response to description given by LEAD. The story and characters are developed by exchange of detailed event descriptions and character dialogs, successively given by both LEAD and ASSOCIATE.' + '\n' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/ChatGLM.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/ChatML.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '<|im_start|>system\n' + '' + '<|im_end|>\n' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Chinese-Vicuna-Chat.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + 'The following is a conversation between an AI assistant called Assistant and a human user called User. The assistant is intelligent, knowledgeable and polite to answer questions of user.' + '\n\n' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Galactica Cite.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Galactica Finetuned.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Galactica Q.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Galactica Summary.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Galactica Work.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Galactica v2.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '<prefix>' + 'You are a helpful chatbot name Stan' + '</prefix>' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Galactica.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Gorilla.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Guanaco non-chat.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Guanaco-QLoRA.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/H2O-prompt_answer.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + '' + '' -}}
   {%- endif %}
   {%- for message in messages %}

instruction-templates/Hippogriff.yaml

@@ -1,11 +1,11 @@
 instruction_template: |-
-  {%- set found_item = false -%}
+  {%- set ns = namespace(found=false) -%}
   {%- for message in messages -%}
       {%- if message['role'] == 'system' -%}
-          {%- set found_item = true -%}
+          {%- set ns.found = true -%}
       {%- endif -%}
   {%- endfor -%}
-  {%- if not found_item -%}
+  {%- if not ns.found -%}
       {{- '' + 'You are a helpful assistant' + '\n' -}}
   {%- endif %}
   {%- for message in messages %}