use together for llama3.1

fastrepl · Aug 19, 2024 · 63e72e8 · 63e72e8
1 parent 6ef69df
commit 63e72e8
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 5 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/berriai/litellm:main-v1.43.10-stable
+FROM ghcr.io/berriai/litellm:main-v1.43.16-stable
 
 WORKDIR /app
 COPY config.yaml .

diff --git a/README.md b/README.md
@@ -11,6 +11,7 @@
 - `AZURE_API_KEY_6`
 - `ANTHROPIC_API_KEY`
 - `ANYSCALE_API_KEY`
+- `TOGETHER_API_KEY`
 - `FIREWORKS_API_KEY`
 - `MASTER_KEY`
 - `REDIS_HOST`

diff --git a/config.yaml b/config.yaml
@@ -90,14 +90,22 @@ model_list:
       api_key: os.environ/ANTHROPIC_API_KEY
       rpm: 1000
       tpm: 40000
+  # - model_name: llama-3.1-8b
+  #   litellm_params:
+  #     model: fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct
+  #     api_key: os.environ/FIREWORKS_API_KEY
+  # - model_name: llama-3.1-70b
+  #   litellm_params:
+  #     model: fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct
+  #     api_key: os.environ/FIREWORKS_API_KEY
   - model_name: llama-3.1-8b
     litellm_params:
-      model: fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct
-      api_key: os.environ/FIREWORKS_API_KEY
+      model: together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
+      api_key: os.environ/TOGETHER_API_KEY
   - model_name: llama-3.1-70b
     litellm_params:
-      model: fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct
-      api_key: os.environ/FIREWORKS_API_KEY
+      model: together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
+      api_key: os.environ/TOGETHER_API_KEY
 litellm_settings:
   drop_params: true
   callbacks: