From fa2d7242b3509fb967e94d222f50ebc3f660f9aa Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 23 Nov 2024 12:03:12 +0000 Subject: [PATCH] update Google Gemini model (#237) --- CHANGELOG.md | 16 +++++++++--- Project.toml | 2 +- src/user_preferences.jl | 11 ++++++--- test_analysis.txt | 54 ----------------------------------------- 4 files changed, 21 insertions(+), 62 deletions(-) delete mode 100644 test_analysis.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 09b686320..b303f1007 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +## [0.65.0] + +### Breaking +- Changed the official ENV variable for MistralAI API from `MISTRALAI_API_KEY` to `MISTRAL_API_KEY` to be compatible with the Mistral docs. + +### Added +- Added a new Gemini Experimental model from November 2024 (`gemini-exp-1121` with alias `gemexp`). + +### Updated +- Changed the ENV variable for MistralAI API from `MISTRALAI_API_KEY` to `MISTRAL_API_KEY` to be compatible with the Mistral docs. + ## [0.64.0] ### Added @@ -618,13 +629,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.4.0] -### Breaking -- Changed the official ENV variable for MistralAI API from `MISTRALAI_API_KEY` to `MISTRAL_API_KEY` to be compatible with the Mistral docs. - ### Added - Improved AICode parsing and error handling (eg, capture more REPL prompts, detect parsing errors earlier, parse more code fence types), including the option to remove unsafe code (eg, `Pkg.add("SomePkg")`) with `AICode(msg; skip_unsafe=true, vebose=true)` - Added new prompt templates: `JuliaRecapTask`, `JuliaRecapCoTTask`, `JuliaExpertTestCode` and updated `JuliaExpertCoTTask` to be more robust against early stopping for smaller OSS models -- Added support for MistralAI API via the MistralOpenAISchema(). All their standard models have been registered, so you should be able to just use `model="mistral-tiny` in your `aigenerate` calls without any further changes. Remember to either provide `api_kwargs.api_key` or ensure you have ENV variable `MISTRAL_API_KEY` set. +- Added support for MistralAI API via the MistralOpenAISchema(). All their standard models have been registered, so you should be able to just use `model="mistral-tiny` in your `aigenerate` calls without any further changes. Remember to either provide `api_kwargs.api_key` or ensure you have ENV variable `MISTRALAI_API_KEY` set. Note: This has been since changed to `MISTRAL_API_KEY` to be compatible with the Mistral docs, refer to the versions after v0.65. - Added support for any OpenAI-compatible API via `schema=CustomOpenAISchema()`. All you have to do is to provide your `api_key` and `url` (base URL of the API) in the `api_kwargs` keyword argument. This option is useful if you use [Perplexity.ai](https://docs.perplexity.ai/), [Fireworks.ai](https://app.fireworks.ai/), or any other similar services. ## [0.3.0] diff --git a/Project.toml b/Project.toml index aa580b51f..0e450fa1b 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.64.0" +version = "0.65.0" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" diff --git a/src/user_preferences.jl b/src/user_preferences.jl index 97df8c479..166754ee5 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -190,8 +190,8 @@ function load_api_keys!() AZURE_OPENAI_HOST = @load_preference("AZURE_OPENAI_HOST", default=get(ENV, "AZURE_OPENAI_HOST", "")) global MISTRAL_API_KEY - MISTRAL_API_KEY = @load_preference("MISTRAL_API_KEY", - default=get(ENV, "MISTRAL_API_KEY", + MISTRAL_API_KEY = @load_preference("MISTRAL_API_KEY", + default=get(ENV, "MISTRAL_API_KEY", get(ENV, "MISTRALAI_API_KEY", ""))) if !isempty(get(ENV, "MISTRALAI_API_KEY", "")) @warn "The MISTRALAI_API_KEY environment variable is deprecated. Use MISTRAL_API_KEY instead." @@ -483,7 +483,7 @@ aliases = merge( "gem15p" => "gemini-1.5-pro-latest", "gem15f8" => "gemini-1.5-flash-8b-latest", "gem15f" => "gemini-1.5-flash-latest", - "gemexp" => "gemini-exp-1114" # latest experimental model from November 2024 + "gemexp" => "gemini-exp-1121" # latest experimental model from November 2024 ), ## Load aliases from preferences as well @load_preference("MODEL_ALIASES", default=Dict{String, String}())) @@ -1130,6 +1130,11 @@ registry = Dict{String, ModelSpec}( 3.0e-7, "Gemini 1.5 Flash is a high-performance model optimized for speed while maintaining strong capabilities across various tasks. 128K context window."), "gemini-exp-1114" => ModelSpec("gemini-exp-1114", + GoogleOpenAISchema(), + 1.25e-6, + 5e-6, + "Gemini Experimental Model from November 2024. Pricing assumed as per Gemini 1.5 Pro. See details [here](https://ai.google.dev/gemini-api/docs/models/experimental-models#use-an-experimental-model)."), + "gemini-exp-1121" => ModelSpec("gemini-exp-1121", GoogleOpenAISchema(), 1.25e-6, 5e-6, diff --git a/test_analysis.txt b/test_analysis.txt deleted file mode 100644 index c13bf3058..000000000 --- a/test_analysis.txt +++ /dev/null @@ -1,54 +0,0 @@ -Test Coverage Analysis for GoogleOpenAISchema - -Current Issues: -- 6 uncovered lines in src/llm_openai_schema_defs.jl -- Patch coverage at 14.29% - -Implementation Analysis: -1. GoogleProvider struct (lines 219-223) - - Basic struct definition, likely covered - - auth_header method (lines 225-227) might be uncovered - -2. create_chat method (lines 229-244) - - Key lines that might be uncovered: - * Line 235: api_key override logic - * Line 237: GoogleProvider instantiation - * Lines 238-243: OpenAI.openai_request call - -3. create_embeddings method (lines 415-429) - - Similar pattern to create_chat - - Potential uncovered lines: - * Line 421: api_key override logic - * Line 422: GoogleProvider instantiation - * Lines 423-428: OpenAI.openai_request call - -Hypotheses for Coverage Issues: -1. Streaming callback paths not tested - - We're using openai_request directly, which might have different behavior - - Solution: Add tests for streaming scenarios - -2. Error handling paths not tested - - No tests for API errors or invalid responses - - Solution: Add tests with mock error responses - -3. Provider instantiation edge cases - - GoogleProvider creation with different URL combinations not tested - - Solution: Add tests with various URL configurations - -4. API key override logic not fully tested - - Need to test all combinations of empty/non-empty GOOGLE_API_KEY - - Solution: Expand current tests to cover more scenarios - -5. Request parameter handling not fully tested - - Different combinations of optional parameters not tested - - Solution: Add tests with various kwargs combinations - -Most Likely Issue: -Hypothesis #4 seems most likely - our tests don't fully exercise the API key override logic in both create_chat and create_embeddings methods. The current tests only check basic scenarios, but we need to test edge cases and different combinations of API keys. - -Action Plan: -1. Add tests for API key override edge cases -2. Add tests for different URL configurations -3. Add tests for error scenarios -4. Add tests for streaming callbacks -5. Add tests for various kwargs combinations